ova_import generates gns3a file to import ova appliance into GNS3
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

ova_import 18KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492
  1. #!/usr/bin/env python3
  2. # Copyright (C) 2016-2018 Bernhard Ehlers
  3. #
  4. # This program is free software: you can redistribute it and/or modify
  5. # it under the terms of the GNU General Public License as published by
  6. # the Free Software Foundation, either version 3 of the License, or
  7. # (at your option) any later version.
  8. #
  9. # This program is distributed in the hope that it will be useful,
  10. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12. # GNU General Public License for more details.
  13. #
  14. # You should have received a copy of the GNU General Public License
  15. # along with this program. If not, see <http://www.gnu.org/licenses/>.
  16. """
  17. ova_import generates gns3a file to import ova appliance into GNS3.
  18. usage: ova_import [-h] [-d DIR] ova
  19. positional arguments:
  20. ova .ova appliance
  21. optional arguments:
  22. -h, --help show this help message and exit
  23. -d DIR, --dir DIR directory for storing gns3a appliance
  24. -u, --uncompress uncompress VMDK
  25. """
  26. import os
  27. import sys
  28. import argparse
  29. import hashlib
  30. import json
  31. import struct
  32. import tarfile
  33. import zlib
  34. import xml.etree.ElementTree as ElementTree
  35. from collections import defaultdict, OrderedDict
  36. def strip_namespace(tag):
  37. """ strip namespace from tag """
  38. if '}' in tag:
  39. tag = tag.split('}', 1)[1] # strip all namespaces
  40. return tag
  41. def etree_to_dict(etree):
  42. """
  43. convert ElementTree to dictionary
  44. see K3---rnc's answer of "Converting xml to dictionary using ElementTree"
  45. http://stackoverflow.com/questions/7684333/converting-xml-to-dictionary-using-elementtree#answer-10076823
  46. variable names changed to make it easier understandable
  47. """
  48. tag = strip_namespace(etree.tag)
  49. node = {tag: {} if etree.attrib else None}
  50. children = list(etree)
  51. if children:
  52. all_child_nodes = defaultdict(list)
  53. for child_node in map(etree_to_dict, children):
  54. for key, val in child_node.items():
  55. all_child_nodes[key].append(val)
  56. node = {tag: {key:val[0] if len(val) == 1 else val
  57. for key, val in all_child_nodes.items()}}
  58. if etree.attrib:
  59. node[tag].update((strip_namespace(key), val)
  60. for key, val in etree.attrib.items())
  61. if etree.text:
  62. text = etree.text.strip()
  63. if children or etree.attrib:
  64. if text:
  65. node[tag]['#text'] = text
  66. else:
  67. node[tag] = text
  68. return node
  69. def create_gns3a(fname, dname, file_info, ovf_data):
  70. """ create GNS3 appliance file """
  71. fbase = os.path.basename(fname)
  72. gns3a = OrderedDict()
  73. ovf = etree_to_dict(ElementTree.fromstring(ovf_data))['Envelope']
  74. # base informations
  75. try:
  76. vm_name = ovf['VirtualSystem']['Name']
  77. except KeyError:
  78. vm_name = ovf['VirtualSystem']['id']
  79. gns3a["name"] = vm_name
  80. gns3a["category"] = "guest"
  81. gns3a["description"] = "ova import of " + fbase
  82. gns3a["vendor_name"] = "unknown"
  83. gns3a["vendor_url"] = "http://www.example.com"
  84. gns3a["product_name"] = vm_name
  85. gns3a["registry_version"] = 3
  86. gns3a["status"] = "experimental"
  87. gns3a["maintainer"] = "GNS3 Team"
  88. gns3a["maintainer_email"] = "developers@gns3.net"
  89. # qemu
  90. cores = 0
  91. eth_adapters = 0
  92. ram = 0
  93. for item in ovf['VirtualSystem']['VirtualHardwareSection']['Item']:
  94. if item['ResourceType'] == '3':
  95. cores = int(item['VirtualQuantity'])
  96. elif item['ResourceType'] == '4':
  97. ram = int(item['VirtualQuantity'])
  98. elif item['ResourceType'] == '10':
  99. eth_adapters += 1
  100. cores = max(cores, 1)
  101. eth_adapters = max(eth_adapters, 1)
  102. if ram == 0:
  103. ram = 256
  104. try:
  105. vm_os = ovf['VirtualSystem']['OperatingSystemSection']['osType']
  106. except KeyError:
  107. vm_os = ovf['VirtualSystem']['OperatingSystemSection']['OSType']['#text']
  108. qemu = OrderedDict()
  109. qemu["adapter_type"] = "e1000"
  110. qemu["adapters"] = eth_adapters
  111. qemu["ram"] = ram
  112. qemu["arch"] = "i386"
  113. if "64" in vm_os:
  114. qemu["arch"] = "x86_64"
  115. qemu["console_type"] = "telnet"
  116. qemu["kvm"] = "allow"
  117. if cores > 1:
  118. qemu["options"] = "-smp " + str(cores)
  119. gns3a["qemu"] = qemu
  120. # images
  121. images = []
  122. for name in sorted(file_info.keys()):
  123. image = OrderedDict()
  124. image["filename"] = name
  125. image["version"] = "0.0"
  126. image["md5sum"] = file_info[name]["md5"]
  127. image["filesize"] = file_info[name]["len"]
  128. images.append(image)
  129. gns3a["images"] = images
  130. # versions
  131. images = OrderedDict()
  132. cdrom = None
  133. disk_id = 0
  134. ovf_ref_file = ovf['References']['File']
  135. if not isinstance(ovf_ref_file, (list, tuple)):
  136. ovf_ref_file = [ovf_ref_file]
  137. for image in ovf_ref_file:
  138. img = image['href']
  139. if img.endswith(".iso"):
  140. if cdrom is None:
  141. cdrom = img
  142. else:
  143. images["hd" + chr(ord("a")+disk_id) + "_disk_image"] = img
  144. disk_id += 1
  145. if cdrom is not None:
  146. images["cdrom_image"] = cdrom
  147. gns3a["versions"] = [OrderedDict([("name", "0.0"), ("images", images)])]
  148. # write to file
  149. ofile = os.path.join(dname, os.path.splitext(fbase)[0] + ".gns3a")
  150. with open(ofile, "w") as f_out:
  151. json.dump(gns3a, f_out, indent=4, separators=(',', ': '))
  152. f_out.write("\n")
  153. SECTOR_SIZE = 512
  154. def div_ceil(dividend, divisor):
  155. """ integer division, round up """
  156. return -(-dividend // divisor)
  157. def copy_raw(f_in, f_out, initial_data=None):
  158. """ copy input file to output file, return file info """
  159. f_len = 0
  160. f_md5 = hashlib.md5()
  161. if initial_data:
  162. f_out.write(initial_data)
  163. f_len += len(initial_data)
  164. f_md5.update(initial_data)
  165. while True:
  166. block = f_in.read(64 * 1024)
  167. if not block:
  168. break
  169. f_out.write(block)
  170. f_len += len(block)
  171. f_md5.update(block)
  172. return {'len': f_len, 'md5': f_md5.hexdigest()}
  173. def copy_vmdk(f_in, f_out, out_name):
  174. """ uncompress VMDK file, return file info """
  175. header = f_in.read(SECTOR_SIZE)
  176. if len(header) != SECTOR_SIZE or header[0:4] != b'KDMV':
  177. # unknown file type: extract unchanged
  178. return copy_raw(f_in, f_out, header)
  179. # decode header
  180. (magic, _, flags, capacity, grain_size, descriptor_offset, descriptor_size,
  181. num_gte_per_gt, _, _, overhead, _, single_eol_char, non_eol_char,
  182. double_eol_char1, double_eol_char2, compress_algorithm) = \
  183. struct.unpack_from('<LLLQQQQLQQQBccccH', header)
  184. if flags & 0x30000 != 0x30000 or compress_algorithm != 1:
  185. # uncompressed VMDK file: extract unchanged
  186. return copy_raw(f_in, f_out, header)
  187. # grain directory / table
  188. gt_size = div_ceil(num_gte_per_gt * 4, SECTOR_SIZE)
  189. gt_count = div_ceil(capacity, grain_size * num_gte_per_gt)
  190. gd_size = div_ceil(gt_count * 4, SECTOR_SIZE)
  191. rgd_offset = 8
  192. gd_offset = rgd_offset + gd_size + gt_count*gt_size
  193. out_overhead = div_ceil(gd_offset + gd_size + gt_count*gt_size, 128) * 128
  194. # read/modify descriptor
  195. if descriptor_offset > 0 and descriptor_size > 0:
  196. skip_len = (descriptor_offset-1) * SECTOR_SIZE
  197. if skip_len > 0 and len(f_in.read(skip_len)) != skip_len:
  198. raise ValueError('Premature EOF')
  199. descriptor = f_in.read(descriptor_size * SECTOR_SIZE)
  200. if len(descriptor) != descriptor_size * SECTOR_SIZE:
  201. raise ValueError('Premature EOF')
  202. in_offset = descriptor_offset + descriptor_size
  203. descriptor = descriptor.rstrip(b'\0')
  204. descriptor = descriptor.replace(double_eol_char1 + double_eol_char2,
  205. single_eol_char)
  206. if descriptor[-1:] != single_eol_char:
  207. descriptor += single_eol_char
  208. while non_eol_char + single_eol_char in descriptor:
  209. descriptor = descriptor.replace(non_eol_char + single_eol_char,
  210. single_eol_char)
  211. descriptor = descriptor.replace(b'"streamOptimized"',
  212. b'"monolithicSparse"')
  213. off_extend = descriptor.find(single_eol_char + b'RDONLY ')
  214. if off_extend < 0:
  215. raise ValueError('No extent defined in descriptor')
  216. off_extend += 1
  217. off_extend_end = descriptor.find(single_eol_char, off_extend)
  218. if off_extend_end < 0:
  219. off_extend_end = len(descriptor)
  220. if descriptor.find(b'RDONLY ', off_extend_end) >= 0:
  221. raise ValueError('Multiple extents defined in descriptor')
  222. descriptor = descriptor[0:off_extend] + \
  223. 'RW {:d} SPARSE "{}"'.format(capacity, out_name).encode('utf-8') + \
  224. descriptor[off_extend_end:]
  225. out_desc_offset = 1
  226. out_desc_size = rgd_offset - 1
  227. if len(descriptor) > out_desc_size * SECTOR_SIZE:
  228. raise ValueError('Descriptor too big')
  229. else:
  230. in_offset = 1
  231. out_desc_offset = 0
  232. out_desc_size = 0
  233. # write new header
  234. header = struct.pack('<LLLQQQQLQQQBccccH', \
  235. magic, 1, 3, capacity, grain_size, out_desc_offset, out_desc_size, \
  236. num_gte_per_gt, rgd_offset, gd_offset, out_overhead, 0, \
  237. single_eol_char, non_eol_char, double_eol_char1, double_eol_char2, 0)
  238. header += b'\0' * (SECTOR_SIZE - len(header))
  239. f_out.write(header)
  240. # write descriptor
  241. if out_desc_offset > 0:
  242. descriptor += b'\0' * (out_desc_size*SECTOR_SIZE - len(descriptor))
  243. f_out.write(descriptor)
  244. # skip to data
  245. skip_len = (overhead - in_offset) * SECTOR_SIZE
  246. if skip_len > 0 and len(f_in.read(skip_len)) != skip_len:
  247. raise ValueError('Premature EOF')
  248. in_offset = overhead
  249. out_offset = out_overhead
  250. f_out.seek(out_offset * SECTOR_SIZE, 0)
  251. # read, uncompress and write data
  252. grain_directory = None
  253. grain_table = []
  254. grain_trans_init = {0:0, 1:1}
  255. grain_translation = grain_trans_init.copy()
  256. gt_translation = {0:-1}
  257. max_datasize = grain_size * SECTOR_SIZE
  258. # add worst case compression overhead
  259. max_datasize += div_ceil(max_datasize, 8) + div_ceil(max_datasize, 64) + 11
  260. while True:
  261. header = f_in.read(SECTOR_SIZE)
  262. if len(header) < SECTOR_SIZE:
  263. raise ValueError('Premature EOF')
  264. (val, size, marker) = struct.unpack_from('<QLL', header)
  265. if size > 0:
  266. # Compressed Grain
  267. if size > max_datasize:
  268. raise ValueError('Bad data block @{:08X}'.format(in_offset))
  269. prev_offset = in_offset
  270. grain_translation[in_offset] = out_offset
  271. data = header[12:]
  272. if size > (SECTOR_SIZE - 12):
  273. data += f_in.read(size - (SECTOR_SIZE - 12))
  274. if len(data) != size:
  275. raise ValueError('Premature EOF')
  276. padding = -(size + 12) % SECTOR_SIZE
  277. if padding > 0 and len(f_in.read(padding)) != padding:
  278. raise ValueError('Premature EOF')
  279. in_offset += div_ceil(size + 12, SECTOR_SIZE)
  280. else:
  281. data = data[0:size]
  282. in_offset += 1
  283. try:
  284. data = zlib.decompress(data)
  285. except zlib.error:
  286. raise ValueError('Bad data block @{:08X}'.format(prev_offset))
  287. if len(data) != grain_size*SECTOR_SIZE:
  288. raise ValueError('Data block @{:08X} has wrong size {:d}'.format(prev_offset, len(data)))
  289. f_out.write(data)
  290. out_offset += grain_size
  291. elif marker == 0:
  292. # End-of-Stream
  293. if f_in.read(SECTOR_SIZE):
  294. raise ValueError('No EOF after end of stream @{:08X}'.format(in_offset))
  295. break
  296. elif marker == 1:
  297. # Grain Table
  298. if len(grain_table) >= gt_count:
  299. raise ValueError('Too many grain tables @{:08X}'.format(in_offset))
  300. if val != gt_size:
  301. raise ValueError('Grain table @{:08X} has wrong size {:d}'.format(in_offset, val * SECTOR_SIZE))
  302. gt_translation[in_offset+1] = len(grain_table)
  303. data = f_in.read(val * SECTOR_SIZE)
  304. if len(data) != val * SECTOR_SIZE:
  305. raise ValueError('Premature EOF')
  306. data = struct.unpack_from("<{:d}L".format(num_gte_per_gt), data)
  307. # apply new offsets in grain tables
  308. try:
  309. data = [grain_translation[x] for x in data]
  310. except KeyError:
  311. raise ValueError('Grain table uses invalid data references')
  312. grain_table.append(data)
  313. grain_translation = grain_trans_init.copy()
  314. in_offset += 1 + val
  315. elif marker == 2:
  316. # Grain Directory
  317. if grain_directory is not None:
  318. raise ValueError('Too many grain directories @{:08X}'.format(in_offset))
  319. if val != gd_size:
  320. raise ValueError('Grain directory @{:08X} has wrong size {:d}'.format(in_offset, val * SECTOR_SIZE))
  321. data = f_in.read(val * SECTOR_SIZE)
  322. if len(data) != val * SECTOR_SIZE:
  323. raise ValueError('Premature EOF')
  324. data = struct.unpack_from("<{:d}L".format(gt_count), data)
  325. grain_directory = data
  326. in_offset += 1 + val
  327. else:
  328. # ignore footer or unknown marker
  329. if val > 0 and len(f_in.read(val*SECTOR_SIZE)) != val*SECTOR_SIZE:
  330. raise ValueError('Premature EOF')
  331. in_offset += 1 + val
  332. # set table index in grain directory
  333. if grain_directory is None:
  334. raise ValueError('No grain directory')
  335. try:
  336. grain_directory = [gt_translation[x] for x in grain_directory]
  337. except KeyError:
  338. raise ValueError('Grain directory uses invalid table references')
  339. # grain directory: sort grain tables, update table index to sector offset
  340. gt_empty = [0] * num_gte_per_gt
  341. unsorted_grain_table = grain_table
  342. grain_table = []
  343. gt_offset = rgd_offset + gd_size
  344. for index in range(0, gt_count):
  345. if grain_directory[index] < 0:
  346. grain_table.append(gt_empty)
  347. else:
  348. grain_table.append(unsorted_grain_table[grain_directory[index]])
  349. grain_directory[index] = gt_offset
  350. gt_offset += gt_size
  351. # save first/redundant grain descriptor
  352. f_out.seek(rgd_offset * SECTOR_SIZE, 0)
  353. data = struct.pack("<{:d}L".format(gt_count), *grain_directory)
  354. data += b'\0' * (gd_size*SECTOR_SIZE - len(data))
  355. f_out.write(data)
  356. for table in grain_table:
  357. data = struct.pack("<{:d}L".format(num_gte_per_gt), *table)
  358. data += b'\0' * (gt_size*SECTOR_SIZE - len(data))
  359. f_out.write(data)
  360. # save second grain descriptor
  361. second_directory = [x - rgd_offset + gd_offset for x in grain_directory]
  362. data = struct.pack("<{:d}L".format(gt_count), *second_directory)
  363. data += b'\0' * (gd_size*SECTOR_SIZE - len(data))
  364. f_out.write(data)
  365. for table in grain_table:
  366. data = struct.pack("<{:d}L".format(num_gte_per_gt), *table)
  367. data += b'\0' * (gt_size*SECTOR_SIZE - len(data))
  368. f_out.write(data)
  369. # get file information
  370. f_out.seek(0, 0)
  371. f_len = 0
  372. f_md5 = hashlib.md5()
  373. while True:
  374. block = f_out.read(64 * 1024)
  375. if not block:
  376. break
  377. f_len += len(block)
  378. f_md5.update(block)
  379. return {'len': f_len, 'md5': f_md5.hexdigest()}
  380. def main(argv):
  381. """ main """
  382. # parse command line
  383. parser = argparse.ArgumentParser(description='%(prog)s generates gns3a file to import ova appliance into GNS3.')
  384. parser.add_argument('-d', '--dir', default='',
  385. help='directory for storing gns3a appliance')
  386. parser.add_argument('-u', '--uncompress', action='store_true',
  387. help='uncompress VMDK')
  388. parser.add_argument('ova', help='.ova appliance')
  389. args = parser.parse_args(argv[1:])
  390. fname = args.ova
  391. dname = args.dir
  392. if dname != '' and not os.path.isdir(dname):
  393. sys.exit("Directory '{}' doesn't exist.".format(dname))
  394. # open ovf file
  395. try:
  396. tar = tarfile.open(fname)
  397. except (IOError, tarfile.TarError) as err:
  398. sys.exit("Error reading ova file: {}".format(err))
  399. # get files from ova
  400. file_info = {}
  401. ovf_data = None
  402. for tarinfo in tar:
  403. name = tarinfo.name.split('/')[-1]
  404. if not tarinfo.isfile(): # ova uses only regular files
  405. pass
  406. elif name.endswith(".ovf"): # ovf file
  407. f_in = tar.extractfile(tarinfo)
  408. ovf_data = f_in.read()
  409. f_in.close()
  410. elif name.endswith(".mf"): # ignore manifest file
  411. pass
  412. else: # save image file
  413. out_name = os.path.join(dname, name)
  414. f_in = tar.extractfile(tarinfo)
  415. f_out = open(out_name, 'w+b')
  416. try:
  417. if args.uncompress and name.endswith(".vmdk"):
  418. # uncompress VMDK file
  419. file_info[name] = copy_vmdk(f_in, f_out, name)
  420. else:
  421. file_info[name] = copy_raw(f_in, f_out)
  422. except ValueError as err:
  423. sys.exit("{}: {}".format(name, err))
  424. f_in.close()
  425. f_out.close()
  426. os.utime(out_name, (tarinfo.mtime, tarinfo.mtime))
  427. tar.close()
  428. if ovf_data is None:
  429. sys.exit("No ovf information in ova.")
  430. create_gns3a(fname, dname, file_info, ovf_data)
  431. if __name__ == "__main__":
  432. main(sys.argv)