gns3feed - generate ATOM/RSS feed from GNS3 community website
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

200 lines
7.6 KiB

  1. #!/usr/bin/env python3
  2. """
  3. gns3feed - generate ATOM/RSS feed from GNS3 community website
  4. usage: gns3feed [-h] [-v] [-a file] [-r file]
  5. optional arguments:
  6. -h, --help, -? prints this screen
  7. -v, --version prints version
  8. -a file, --atom file generate ATOM feed
  9. -r file, --rss file generate RSS feed
  10. """
  11. __version__ = "0.2"
  12. import os
  13. import sys
  14. import argparse
  15. import hashlib
  16. import re
  17. import datetime
  18. import dateutil.parser
  19. import urllib3
  20. import requests
  21. from feedgen.feed import FeedGenerator
  22. HOST = "https://gns3.com"
  23. def requests_session_retry(retries=2, backoff=0.3):
  24. """ open requests session with retry parameter """
  25. session = requests.Session()
  26. retry = urllib3.util.retry.Retry(total=retries, backoff_factor=backoff)
  27. adapter = requests.adapters.HTTPAdapter(max_retries=retry)
  28. session.mount('http://', adapter)
  29. session.mount('https://', adapter)
  30. return session
  31. def get_data(session, url, data):
  32. """ get data from GNS3 website """
  33. headers = {'User-Agent': 'gns3feed/' + __version__,
  34. 'Content-Type': 'application/json',
  35. 'Referer': HOST+"/"}
  36. try:
  37. resp = session.post(HOST + url, headers=headers, timeout=30,
  38. json=data)
  39. resp.raise_for_status()
  40. if not resp.headers['Content-Type'].startswith('application/json'):
  41. sys.exit("{} - Unexpected content type {}".format(
  42. url, resp.headers['Content-Type']))
  43. data = resp.json()
  44. except (requests.exceptions.RequestException, ValueError) as err:
  45. msg = str(err)
  46. match = re.search(r"\(Caused by ([a-zA-Z0-9_]+)\('[^:]*: (.*)'\)", msg)
  47. if match:
  48. msg = "{}: {} for host: {}".format(*match.groups(), HOST)
  49. sys.exit("{} - {}".format(url, msg))
  50. return data
  51. def date_id(date):
  52. """ create an id from a date """
  53. return date.astimezone(datetime.timezone.utc).strftime('%Y%m%d%H%M%S')
  54. INVALID_CHARS = re.compile("[" \
  55. "\u0000-\u0008\u000B\u000E-\u001F\u007F-\u009F"
  56. "\uD800-\uDFFF\uFDD0-\uFDEF\uFFFE\uFFFF"
  57. "\U0001FFFE\U0001FFFF\U0002FFFE\U0002FFFF\U0003FFFE\U0003FFFF"
  58. "\U0004FFFE\U0004FFFF\U0005FFFE\U0005FFFF\U0006FFFE\U0006FFFF"
  59. "\U0007FFFE\U0007FFFF\U0008FFFE\U0008FFFF\U0009FFFE\U0009FFFF"
  60. "\U000AFFFE\U000AFFFF\U000BFFFE\U000BFFFF\U000CFFFE\U000CFFFF"
  61. "\U000DFFFE\U000DFFFF\U000EFFFE\U000EFFFF\U000FFFFE\U000FFFFF"
  62. "\U0010FFFE\U0010FFFF]")
  63. REPLACEMENT_CHAR = "\uFFFD"
  64. def clean_string(string):
  65. """ clean a string from invalid characters """
  66. return INVALID_CHARS.sub(REPLACEMENT_CHAR, string)
  67. def clean_content(content):
  68. """ clean the content of a post """
  69. content = clean_string(content)
  70. content = re.sub(r'<p>\s*</p>[ \t]*', "", content)
  71. content = re.sub(r'(</p>\s*)<br */?>[ \t]*', r'\1', content)
  72. return content
  73. ARTICLE_LIST_PARAMS = {
  74. "operationName": "getInitiativeList",
  75. "variables": {
  76. "limit": 50,
  77. "archived": "exclude",
  78. "sortBy": "dateLastUpdated",
  79. "typeIds": [
  80. "5cc2471c7979fb3b5d34a6cd",
  81. "5e47720e5f1d753f4500ec92",
  82. "5ebe1b516c92855b3c810e20"
  83. ]
  84. },
  85. "query": 'query getInitiativeList($archived: InitiativeListArchivedOptions, $start: ID, $limit: Int = 50, $sortBy: InitiativeListSortOptions, $typeIds: [ID!]) {initiativeList(archived: $archived, start: $start, limit: $limit, sortBy: $sortBy, typeIds: $typeIds) {items {...InitiativeListItemFragment __typename} __typename}}\nfragment InitiativeListItemFragment on Initiative {id addedBy {id fullName __typename} archived body {summary __typename} class dateTimeAdded: dateAdded(format: "YYYY-MM-DDTHH:mm:ssZ") dateTimeLastUpdated: dateLastUpdated(format: "YYYY-MM-DDTHH:mm:ssZ") discussionFormat discussionStatus {closed __typename} labels {id color name __typename} name numNotes numViews pinned privacy slug thread {id lastEntryDateTime: lastEntryDate(format: "YYYY-MM-DDTHH:mm:ssZ") numEntries __typename} type {id slug __typename} __typename}'
  86. }
  87. def add_articles(feed, session):
  88. """ add latest articles """
  89. articles = get_data(session, "/api/graphql", ARTICLE_LIST_PARAMS)
  90. articles = articles['data']['initiativeList']['items']
  91. for article in articles:
  92. if article.get('thread') and \
  93. article['thread'].get('lastEntryDateTime'):
  94. last_modified = article['thread']['lastEntryDateTime']
  95. else:
  96. last_modified = article['dateTimeAdded']
  97. article['_last_modified'] = dateutil.parser.parse(last_modified)
  98. articles = sorted(articles, key=lambda k: k['_last_modified'],
  99. reverse=True)[0:-20]
  100. for article in articles:
  101. title = article['name']
  102. if article.get('discussionStatus') and \
  103. article['discussionStatus'].get('closed'):
  104. article['_last_modified'] += datetime.timedelta(seconds=1)
  105. title = "[Solved] " + title
  106. if article.get('thread') and \
  107. article['thread']['numEntries'] > 0:
  108. title += " ({})".format(article['thread']['numEntries'])
  109. feed_entry = feed.add_entry(order='append')
  110. feed_entry.id(HOST + '/feed/article/' + article['id'] + \
  111. '-' + date_id(article['_last_modified']))
  112. feed_entry.title(title)
  113. feed_entry.published(article['_last_modified'])
  114. feed_entry.updated(article['_last_modified'])
  115. feed_entry.author(name=article['addedBy']['fullName'])
  116. feed_entry.dc.dc_creator(article['addedBy']['fullName'])
  117. feed_entry.content(content=clean_content(article['body']['summary']),
  118. type="html")
  119. feed_entry.link(href="/".join((HOST, "community",
  120. article['type']['slug'],
  121. article['slug'])),
  122. rel='alternate')
  123. def generate_feed(atom_file, rss_file):
  124. """ generate ATOM/RSS feed """
  125. feed = FeedGenerator()
  126. feed.load_extension("dc", atom=False, rss=True)
  127. feed.title("GNS3 Discussions")
  128. feed.description("GNS3 Discussions")
  129. feed.link(href=HOST+"/community/featured", rel='alternate')
  130. feed.language('en')
  131. sess = requests_session_retry()
  132. add_articles(feed, sess)
  133. md5_hash = hashlib.md5()
  134. for feed_entry in feed.entry():
  135. md5_hash.update(feed_entry.id().encode('utf-8'))
  136. feed.id(HOST + '/feed/' + md5_hash.hexdigest())
  137. try:
  138. if atom_file:
  139. feed_type = "ATOM"
  140. atom_file = os.path.expanduser(atom_file)
  141. feed.atom_file(atom_file + ".new")
  142. os.replace(atom_file + ".new", atom_file)
  143. if rss_file:
  144. feed_type = "RSS"
  145. rss_file = os.path.expanduser(rss_file)
  146. feed.rss_file(rss_file + ".new")
  147. os.replace(rss_file + ".new", rss_file)
  148. except OSError as err:
  149. sys.exit("Can't create {} feed: {}".format(feed_type, err))
  150. # Main
  151. parser = argparse.ArgumentParser(add_help=False, \
  152. description='%(prog)s - generate ATOM/RSS feed from GNS3 community website')
  153. parser.add_argument('-h', '--help', '-?', action='help',
  154. help='prints this screen')
  155. parser.add_argument('-v', '--version', action='version',
  156. version="%(prog)s v" + __version__,
  157. help='prints version')
  158. parser.add_argument('-a', '--atom', metavar="file",
  159. help='generate ATOM feed')
  160. parser.add_argument('-r', '--rss', metavar="file",
  161. help='generate RSS feed')
  162. args = parser.parse_args()
  163. if not args.atom and not args.rss:
  164. parser.error("at least one of the options -a/--atom or -r/--rss must be used.")
  165. generate_feed(args.atom, args.rss)