ova_import generates gns3a file to import ova appliance into GNS3
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

517 lines
18 KiB

  1. #!/usr/bin/env python3
  2. # Copyright (C) 2016-2018 Bernhard Ehlers
  3. #
  4. # This program is free software: you can redistribute it and/or modify
  5. # it under the terms of the GNU General Public License as published by
  6. # the Free Software Foundation, either version 3 of the License, or
  7. # (at your option) any later version.
  8. #
  9. # This program is distributed in the hope that it will be useful,
  10. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12. # GNU General Public License for more details.
  13. #
  14. # You should have received a copy of the GNU General Public License
  15. # along with this program. If not, see <http://www.gnu.org/licenses/>.
  16. """
  17. ova_import generates gns3a file to import ova appliance into GNS3.
  18. usage: ova_import [-h] [-d DIR] [-k] ova
  19. positional arguments:
  20. ova .ova appliance
  21. optional arguments:
  22. -h, --help show this help message and exit
  23. -d DIR, --dir DIR directory for storing appliance and images
  24. -k, --keepcompressed keep images compressed
  25. """
  26. import os
  27. import sys
  28. import argparse
  29. import hashlib
  30. import json
  31. import re
  32. import struct
  33. import tarfile
  34. import zlib
  35. import xml.etree.ElementTree as ElementTree
  36. from collections import defaultdict, OrderedDict
  37. def strip_namespace(tag):
  38. """ strip namespace from tag """
  39. if '}' in tag:
  40. tag = tag.split('}', 1)[1] # strip all namespaces
  41. return tag
  42. def etree_to_dict(etree):
  43. """
  44. convert ElementTree to dictionary
  45. see K3---rnc's answer of "Converting xml to dictionary using ElementTree"
  46. http://stackoverflow.com/questions/7684333/converting-xml-to-dictionary-using-elementtree#answer-10076823
  47. variable names changed to make it easier understandable
  48. """
  49. tag = strip_namespace(etree.tag)
  50. node = {tag: {} if etree.attrib else None}
  51. children = list(etree)
  52. if children:
  53. all_child_nodes = defaultdict(list)
  54. for child_node in map(etree_to_dict, children):
  55. for key, val in child_node.items():
  56. all_child_nodes[key].append(val)
  57. node = {tag: {key:val[0] if len(val) == 1 else val
  58. for key, val in all_child_nodes.items()}}
  59. if etree.attrib:
  60. node[tag].update((strip_namespace(key), val)
  61. for key, val in etree.attrib.items())
  62. if etree.text:
  63. text = etree.text.strip()
  64. if children or etree.attrib:
  65. if text:
  66. node[tag]['#text'] = text
  67. else:
  68. node[tag] = text
  69. return node
  70. def create_gns3a(fname, dname, file_info, ovf_data):
  71. """ create GNS3 appliance file """
  72. fbase = os.path.basename(fname)
  73. gns3a = OrderedDict()
  74. ovf = etree_to_dict(ElementTree.fromstring(ovf_data))['Envelope']
  75. # base informations
  76. try:
  77. vm_name = ovf['VirtualSystem']['Name']
  78. except KeyError:
  79. vm_name = ovf['VirtualSystem']['id']
  80. gns3a["name"] = vm_name
  81. gns3a["category"] = "guest"
  82. gns3a["description"] = "ova import of " + fbase
  83. gns3a["vendor_name"] = "unknown"
  84. gns3a["vendor_url"] = "http://www.example.com"
  85. gns3a["product_name"] = vm_name
  86. gns3a["registry_version"] = 3
  87. gns3a["status"] = "experimental"
  88. gns3a["maintainer"] = "GNS3 Team"
  89. gns3a["maintainer_email"] = "developers@gns3.net"
  90. # qemu
  91. cores = 0
  92. eth_adapters = 0
  93. ram = 0
  94. for item in ovf['VirtualSystem']['VirtualHardwareSection']['Item']:
  95. if item['ResourceType'] == '3':
  96. cores = int(item['VirtualQuantity'])
  97. elif item['ResourceType'] == '4':
  98. ram = int(item['VirtualQuantity'])
  99. elif item['ResourceType'] == '10':
  100. eth_adapters += 1
  101. cores = max(cores, 1)
  102. eth_adapters = max(eth_adapters, 1)
  103. if ram == 0:
  104. ram = 256
  105. try:
  106. vm_os = ovf['VirtualSystem']['OperatingSystemSection']['osType']
  107. except KeyError:
  108. vm_os = ovf['VirtualSystem']['OperatingSystemSection']['OSType']['#text']
  109. qemu = OrderedDict()
  110. qemu["adapter_type"] = "e1000"
  111. qemu["adapters"] = eth_adapters
  112. qemu["ram"] = ram
  113. qemu["arch"] = "i386"
  114. if "64" in vm_os:
  115. qemu["arch"] = "x86_64"
  116. qemu["console_type"] = "telnet"
  117. qemu["kvm"] = "allow"
  118. if cores > 1:
  119. qemu["options"] = "-smp " + str(cores)
  120. gns3a["qemu"] = qemu
  121. # images
  122. images = []
  123. for name in sorted(file_info.keys()):
  124. image = OrderedDict()
  125. image["filename"] = name
  126. image["version"] = "0.0"
  127. image["md5sum"] = file_info[name]["md5"]
  128. image["filesize"] = file_info[name]["len"]
  129. images.append(image)
  130. gns3a["images"] = images
  131. # versions
  132. images = OrderedDict()
  133. cdrom = None
  134. disk_id = 0
  135. ovf_ref_file = ovf['References']['File']
  136. if not isinstance(ovf_ref_file, (list, tuple)):
  137. ovf_ref_file = [ovf_ref_file]
  138. for image in ovf_ref_file:
  139. img = image['href']
  140. if img.endswith(".iso"):
  141. if cdrom is None:
  142. cdrom = img
  143. else:
  144. images["hd" + chr(ord("a")+disk_id) + "_disk_image"] = img
  145. disk_id += 1
  146. if cdrom is not None:
  147. images["cdrom_image"] = cdrom
  148. gns3a["versions"] = [OrderedDict([("name", "0.0"), ("images", images)])]
  149. # write to file
  150. ofile = os.path.join(dname, os.path.splitext(fbase)[0] + ".gns3a")
  151. with open(ofile, "w") as f_out:
  152. json.dump(gns3a, f_out, indent=4, separators=(',', ': '))
  153. f_out.write("\n")
  154. SECTOR_SIZE = 512
  155. def div_ceil(dividend, divisor):
  156. """ integer division, round up """
  157. return -(-dividend // divisor)
  158. def copy_raw(f_in, f_out, initial_data=None):
  159. """ copy input file to output file, return file info """
  160. f_len = 0
  161. f_md5 = hashlib.md5()
  162. if initial_data:
  163. f_out.write(initial_data)
  164. f_len += len(initial_data)
  165. f_md5.update(initial_data)
  166. while True:
  167. block = f_in.read(64 * 1024)
  168. if not block:
  169. break
  170. f_out.write(block)
  171. f_len += len(block)
  172. f_md5.update(block)
  173. return {'len': f_len, 'md5': f_md5.hexdigest()}
  174. def read_block(file, datalen):
  175. """ read a block of data from file """
  176. if datalen < 0:
  177. raise ValueError("Can't seek backwards")
  178. if datalen == 0:
  179. data = b''
  180. else:
  181. data = file.read(datalen)
  182. if len(data) != datalen:
  183. raise ValueError('Premature EOF')
  184. return data
  185. def copy_vmdk(f_in, f_out, out_name):
  186. """
  187. uncompress VMDK file, return file info
  188. The VMDK disk format is documented in
  189. https://www.vmware.com/support/developer/vddk/vmdk_50_technote.pdf
  190. """
  191. header = f_in.read(SECTOR_SIZE)
  192. if len(header) != SECTOR_SIZE or header[0:4] != b'KDMV':
  193. # unknown file type: extract unchanged
  194. return copy_raw(f_in, f_out, header)
  195. # decode header
  196. (magic, _, flags, capacity, grain_size, descriptor_offset, descriptor_size,
  197. num_gte_per_gt, _, in_gd_off, overhead, _, single_eol_char, non_eol_char,
  198. double_eol_char1, double_eol_char2, compress_algorithm) = \
  199. struct.unpack_from('<LLLQQQQLQQQBccccH', header)
  200. if flags & 0x30000 != 0x30000 or compress_algorithm != 1:
  201. # uncompressed VMDK file: extract unchanged
  202. return copy_raw(f_in, f_out, header)
  203. # grain directory / table
  204. gt_size = div_ceil(num_gte_per_gt * 4, SECTOR_SIZE)
  205. gt_count = div_ceil(capacity, grain_size * num_gte_per_gt)
  206. gd_size = div_ceil(gt_count * 4, SECTOR_SIZE)
  207. rgd_offset = 8
  208. gd_offset = rgd_offset + gd_size + gt_count*gt_size
  209. out_overhead = div_ceil(gd_offset + gd_size + gt_count*gt_size, 128) * 128
  210. # read/modify descriptor
  211. if descriptor_offset > 0 and descriptor_size > 0:
  212. read_block(f_in, (descriptor_offset-1) * SECTOR_SIZE)
  213. descriptor = read_block(f_in, descriptor_size * SECTOR_SIZE)
  214. in_offset = descriptor_offset + descriptor_size
  215. descriptor = descriptor.rstrip(b'\0')
  216. descriptor = descriptor.replace(double_eol_char1 + double_eol_char2,
  217. single_eol_char)
  218. if single_eol_char != b'\n':
  219. descriptor = descriptor.replace(single_eol_char, b'\n')
  220. descriptor = re.sub(br'^([ \t]*createType[ \t]*=[ \t]*)"streamOptimized"',
  221. br'\1"monolithicSparse"',
  222. descriptor, flags=re.MULTILINE)
  223. descriptor, count = re.subn(br'^([ \t]*)(RW|RDONLY)[ \t].*',
  224. r'\1RW {:d} SPARSE "{}"'.format(capacity, out_name).encode(),
  225. descriptor, flags=re.MULTILINE)
  226. if count == 0:
  227. raise ValueError('No extent defined in descriptor')
  228. if count > 1:
  229. raise ValueError('Multiple extents defined in descriptor')
  230. if single_eol_char != b'\n':
  231. descriptor = descriptor.replace(b'\n', single_eol_char)
  232. out_desc_offset = 1
  233. out_desc_size = rgd_offset - 1
  234. if len(descriptor) > out_desc_size * SECTOR_SIZE:
  235. raise ValueError('Descriptor too big')
  236. else:
  237. in_offset = 1
  238. out_desc_offset = 0
  239. out_desc_size = 0
  240. # write new header
  241. header = struct.pack('<LLLQQQQLQQQBccccH', \
  242. magic, 1, 3, capacity, grain_size, out_desc_offset, out_desc_size, \
  243. num_gte_per_gt, rgd_offset, gd_offset, out_overhead, 0, \
  244. single_eol_char, non_eol_char, double_eol_char1, double_eol_char2, 0)
  245. header += b'\0' * (SECTOR_SIZE - len(header))
  246. f_out.write(header)
  247. # write descriptor
  248. if out_desc_offset > 0:
  249. descriptor += b'\0' * (out_desc_size*SECTOR_SIZE - len(descriptor))
  250. f_out.write(descriptor)
  251. # initialize grain directory and tables
  252. grain_directory = None
  253. grain_table = []
  254. grain_translation = {0:0, 1:1}
  255. gt_translation = {0:-1}
  256. # if available read grain directory and grain tables
  257. if in_gd_off != 0xffffffffffffffff:
  258. if in_gd_off < in_offset or \
  259. in_gd_off + gd_size + gt_count*gt_size > overhead:
  260. raise ValueError('Invalid grain directory offset')
  261. read_block(f_in, (in_gd_off - in_offset) * SECTOR_SIZE)
  262. in_offset = in_gd_off
  263. data = read_block(f_in, gd_size * SECTOR_SIZE)
  264. data = struct.unpack_from("<{:d}L".format(gt_count), data)
  265. grain_directory = data
  266. in_offset += gd_size
  267. for index in range(0, gt_count):
  268. gt_translation[in_offset] = index
  269. data = read_block(f_in, gt_size * SECTOR_SIZE)
  270. data = struct.unpack_from("<{:d}L".format(num_gte_per_gt), data)
  271. grain_table.append(data)
  272. in_offset += gt_size
  273. # skip to data
  274. read_block(f_in, (overhead - in_offset) * SECTOR_SIZE)
  275. in_offset = overhead
  276. out_offset = out_overhead
  277. f_out.seek(out_offset * SECTOR_SIZE, 0)
  278. # read, uncompress and write data
  279. max_datasize = grain_size * SECTOR_SIZE
  280. # add worst case compression overhead
  281. max_datasize += div_ceil(max_datasize, 8) + div_ceil(max_datasize, 64) + 11
  282. while True:
  283. header = read_block(f_in, SECTOR_SIZE)
  284. (val, size, marker) = struct.unpack_from('<QLL', header)
  285. if size > 0:
  286. # Compressed Grain
  287. if size > max_datasize:
  288. raise ValueError('Bad data block @{:08X}'.format(in_offset))
  289. prev_offset = in_offset
  290. grain_translation[in_offset] = out_offset
  291. data = header[12:]
  292. if size > (SECTOR_SIZE - 12):
  293. data += read_block(f_in, size - (SECTOR_SIZE - 12))
  294. read_block(f_in, -(size + 12) % SECTOR_SIZE)
  295. in_offset += div_ceil(size + 12, SECTOR_SIZE)
  296. else:
  297. data = data[0:size]
  298. in_offset += 1
  299. try:
  300. data = zlib.decompress(data)
  301. except zlib.error:
  302. raise ValueError('Bad data block @{:08X}'.format(prev_offset))
  303. if len(data) != grain_size*SECTOR_SIZE:
  304. raise ValueError('Data block @{:08X} has wrong size {:d}'.format(prev_offset, len(data)))
  305. f_out.write(data)
  306. out_offset += grain_size
  307. elif marker == 0:
  308. # End-of-Stream
  309. if f_in.read(SECTOR_SIZE):
  310. raise ValueError('No EOF after end of stream @{:08X}'.format(in_offset))
  311. break
  312. elif marker == 1:
  313. # Grain Table
  314. if len(grain_table) >= gt_count:
  315. raise ValueError('Too many grain tables @{:08X}'.format(in_offset))
  316. if val != gt_size:
  317. raise ValueError('Grain table @{:08X} has wrong size {:d}'.format(in_offset, val * SECTOR_SIZE))
  318. gt_translation[in_offset+1] = len(grain_table)
  319. data = read_block(f_in, val * SECTOR_SIZE)
  320. data = struct.unpack_from("<{:d}L".format(num_gte_per_gt), data)
  321. grain_table.append(data)
  322. in_offset += 1 + val
  323. elif marker == 2:
  324. # Grain Directory
  325. if grain_directory is not None:
  326. raise ValueError('Too many grain directories @{:08X}'.format(in_offset))
  327. if val != gd_size:
  328. raise ValueError('Grain directory @{:08X} has wrong size {:d}'.format(in_offset, val * SECTOR_SIZE))
  329. data = read_block(f_in, val * SECTOR_SIZE)
  330. data = struct.unpack_from("<{:d}L".format(gt_count), data)
  331. grain_directory = data
  332. in_offset += 1 + val
  333. else:
  334. # ignore footer or unknown marker
  335. read_block(f_in, val * SECTOR_SIZE)
  336. in_offset += 1 + val
  337. # set table index in grain directory
  338. if grain_directory is None:
  339. raise ValueError('No grain directory')
  340. try:
  341. grain_directory = [gt_translation[x] for x in grain_directory]
  342. except KeyError:
  343. raise ValueError('Grain directory uses invalid table references')
  344. # apply new offsets in grain tables
  345. for index, table in enumerate(grain_table):
  346. try:
  347. grain_table[index] = [grain_translation[x] for x in table]
  348. except KeyError:
  349. raise ValueError('Grain table uses invalid data references')
  350. # grain directory: sort grain tables, update table index to sector offset
  351. gt_empty = [0] * num_gte_per_gt
  352. unsorted_grain_table = grain_table
  353. grain_table = []
  354. gt_offset = rgd_offset + gd_size
  355. for index in range(0, gt_count):
  356. if grain_directory[index] < 0:
  357. grain_table.append(gt_empty)
  358. else:
  359. grain_table.append(unsorted_grain_table[grain_directory[index]])
  360. grain_directory[index] = gt_offset
  361. gt_offset += gt_size
  362. # save first/redundant grain descriptor
  363. f_out.seek(rgd_offset * SECTOR_SIZE, 0)
  364. data = struct.pack("<{:d}L".format(gt_count), *grain_directory)
  365. data += b'\0' * (gd_size*SECTOR_SIZE - len(data))
  366. f_out.write(data)
  367. for table in grain_table:
  368. data = struct.pack("<{:d}L".format(num_gte_per_gt), *table)
  369. data += b'\0' * (gt_size*SECTOR_SIZE - len(data))
  370. f_out.write(data)
  371. # save second grain descriptor
  372. second_directory = [x - rgd_offset + gd_offset for x in grain_directory]
  373. data = struct.pack("<{:d}L".format(gt_count), *second_directory)
  374. data += b'\0' * (gd_size*SECTOR_SIZE - len(data))
  375. f_out.write(data)
  376. for table in grain_table:
  377. data = struct.pack("<{:d}L".format(num_gte_per_gt), *table)
  378. data += b'\0' * (gt_size*SECTOR_SIZE - len(data))
  379. f_out.write(data)
  380. # get file information
  381. f_out.seek(0, 0)
  382. f_len = 0
  383. f_md5 = hashlib.md5()
  384. while True:
  385. block = f_out.read(64 * 1024)
  386. if not block:
  387. break
  388. f_len += len(block)
  389. f_md5.update(block)
  390. return {'len': f_len, 'md5': f_md5.hexdigest()}
  391. def main(argv):
  392. """ main """
  393. # parse command line
  394. parser = argparse.ArgumentParser(description='%(prog)s generates gns3a file to import ova appliance into GNS3.')
  395. parser.add_argument('-d', '--dir', default='',
  396. help='directory for storing appliance and images')
  397. parser.add_argument('-k', '--keepcompressed', action='store_true',
  398. help='keep images compressed')
  399. parser.add_argument('ova', help='.ova appliance')
  400. args = parser.parse_args(argv[1:])
  401. fname = args.ova
  402. dname = args.dir
  403. if dname != '' and not os.path.isdir(dname):
  404. sys.exit("Directory '{}' doesn't exist.".format(dname))
  405. # open ovf file
  406. try:
  407. tar = tarfile.open(fname)
  408. except (IOError, tarfile.TarError) as err:
  409. sys.exit("Error reading ova file: {}".format(err))
  410. # get files from ova
  411. file_info = {}
  412. ovf_data = None
  413. for tarinfo in tar:
  414. name = tarinfo.name.split('/')[-1]
  415. if not tarinfo.isfile(): # ova uses only regular files
  416. pass
  417. elif name.endswith(".ovf"): # ovf file
  418. f_in = tar.extractfile(tarinfo)
  419. ovf_data = f_in.read()
  420. f_in.close()
  421. elif name.endswith(".mf"): # ignore manifest file
  422. pass
  423. elif name.endswith(".cert"): # ignore certificate
  424. pass
  425. elif name.endswith(".pem"): # ignore certificate
  426. pass
  427. else: # save image file
  428. print("Extracting image {}...".format(name))
  429. out_name = os.path.join(dname, name)
  430. f_in = tar.extractfile(tarinfo)
  431. f_out = open(out_name, 'w+b')
  432. try:
  433. if not args.keepcompressed and name.endswith(".vmdk"):
  434. # uncompress VMDK file
  435. file_info[name] = copy_vmdk(f_in, f_out, name)
  436. else:
  437. file_info[name] = copy_raw(f_in, f_out)
  438. except ValueError as err:
  439. sys.exit("{}: {}".format(name, err))
  440. f_in.close()
  441. f_out.close()
  442. os.utime(out_name, (tarinfo.mtime, tarinfo.mtime))
  443. tar.close()
  444. if ovf_data is None:
  445. sys.exit("No ovf information in ova.")
  446. print("Creating appliance {}..."
  447. .format(os.path.splitext(os.path.basename(fname))[0] + ".gns3a"))
  448. create_gns3a(fname, dname, file_info, ovf_data)
  449. if __name__ == "__main__":
  450. main(sys.argv)