gns3feed - generate ATOM/RSS feed from GNS3 community website
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 

199 lines
7.6 KiB

#!/usr/bin/env python3
"""
gns3feed - generate ATOM/RSS feed from GNS3 community website
usage: gns3feed [-h] [-v] [-a file] [-r file]
optional arguments:
-h, --help, -? prints this screen
-v, --version prints version
-a file, --atom file generate ATOM feed
-r file, --rss file generate RSS feed
"""
__version__ = "0.2"
import os
import sys
import argparse
import hashlib
import re
import datetime
import dateutil.parser
import urllib3
import requests
from feedgen.feed import FeedGenerator
HOST = "https://gns3.com"
def requests_session_retry(retries=2, backoff=0.3):
""" open requests session with retry parameter """
session = requests.Session()
retry = urllib3.util.retry.Retry(total=retries, backoff_factor=backoff)
adapter = requests.adapters.HTTPAdapter(max_retries=retry)
session.mount('http://', adapter)
session.mount('https://', adapter)
return session
def get_data(session, url, data):
""" get data from GNS3 website """
headers = {'User-Agent': 'gns3feed/' + __version__,
'Content-Type': 'application/json',
'Referer': HOST+"/"}
try:
resp = session.post(HOST + url, headers=headers, timeout=30,
json=data)
resp.raise_for_status()
if not resp.headers['Content-Type'].startswith('application/json'):
sys.exit("{} - Unexpected content type {}".format(
url, resp.headers['Content-Type']))
data = resp.json()
except (requests.exceptions.RequestException, ValueError) as err:
msg = str(err)
match = re.search(r"\(Caused by ([a-zA-Z0-9_]+)\('[^:]*: (.*)'\)", msg)
if match:
msg = "{}: {} for host: {}".format(*match.groups(), HOST)
sys.exit("{} - {}".format(url, msg))
return data
def date_id(date):
""" create an id from a date """
return date.astimezone(datetime.timezone.utc).strftime('%Y%m%d%H%M%S')
INVALID_CHARS = re.compile("[" \
"\u0000-\u0008\u000B\u000E-\u001F\u007F-\u009F"
"\uD800-\uDFFF\uFDD0-\uFDEF\uFFFE\uFFFF"
"\U0001FFFE\U0001FFFF\U0002FFFE\U0002FFFF\U0003FFFE\U0003FFFF"
"\U0004FFFE\U0004FFFF\U0005FFFE\U0005FFFF\U0006FFFE\U0006FFFF"
"\U0007FFFE\U0007FFFF\U0008FFFE\U0008FFFF\U0009FFFE\U0009FFFF"
"\U000AFFFE\U000AFFFF\U000BFFFE\U000BFFFF\U000CFFFE\U000CFFFF"
"\U000DFFFE\U000DFFFF\U000EFFFE\U000EFFFF\U000FFFFE\U000FFFFF"
"\U0010FFFE\U0010FFFF]")
REPLACEMENT_CHAR = "\uFFFD"
def clean_string(string):
""" clean a string from invalid characters """
return INVALID_CHARS.sub(REPLACEMENT_CHAR, string)
def clean_content(content):
""" clean the content of a post """
content = clean_string(content)
content = re.sub(r'<p>\s*</p>[ \t]*', "", content)
content = re.sub(r'(</p>\s*)<br */?>[ \t]*', r'\1', content)
return content
ARTICLE_LIST_PARAMS = {
"operationName": "getInitiativeList",
"variables": {
"limit": 50,
"archived": "exclude",
"sortBy": "dateLastUpdated",
"typeIds": [
"5cc2471c7979fb3b5d34a6cd",
"5e47720e5f1d753f4500ec92",
"5ebe1b516c92855b3c810e20"
]
},
"query": 'query getInitiativeList($archived: InitiativeListArchivedOptions, $start: ID, $limit: Int = 50, $sortBy: InitiativeListSortOptions, $typeIds: [ID!]) {initiativeList(archived: $archived, start: $start, limit: $limit, sortBy: $sortBy, typeIds: $typeIds) {items {...InitiativeListItemFragment __typename} __typename}}\nfragment InitiativeListItemFragment on Initiative {id addedBy {id fullName __typename} archived body {summary __typename} class dateTimeAdded: dateAdded(format: "YYYY-MM-DDTHH:mm:ssZ") dateTimeLastUpdated: dateLastUpdated(format: "YYYY-MM-DDTHH:mm:ssZ") discussionFormat discussionStatus {closed __typename} labels {id color name __typename} name numNotes numViews pinned privacy slug thread {id lastEntryDateTime: lastEntryDate(format: "YYYY-MM-DDTHH:mm:ssZ") numEntries __typename} type {id slug __typename} __typename}'
}
def add_articles(feed, session):
""" add latest articles """
articles = get_data(session, "/api/graphql", ARTICLE_LIST_PARAMS)
articles = articles['data']['initiativeList']['items']
for article in articles:
if article.get('thread') and \
article['thread'].get('lastEntryDateTime'):
last_modified = article['thread']['lastEntryDateTime']
else:
last_modified = article['dateTimeAdded']
article['_last_modified'] = dateutil.parser.parse(last_modified)
articles = sorted(articles, key=lambda k: k['_last_modified'],
reverse=True)[0:-20]
for article in articles:
title = article['name']
if article.get('discussionStatus') and \
article['discussionStatus'].get('closed'):
article['_last_modified'] += datetime.timedelta(seconds=1)
title = "[Solved] " + title
if article.get('thread') and \
article['thread']['numEntries'] > 0:
title += " ({})".format(article['thread']['numEntries'])
feed_entry = feed.add_entry(order='append')
feed_entry.id(HOST + '/feed/article/' + article['id'] + \
'-' + date_id(article['_last_modified']))
feed_entry.title(title)
feed_entry.published(article['_last_modified'])
feed_entry.updated(article['_last_modified'])
feed_entry.author(name=article['addedBy']['fullName'])
feed_entry.dc.dc_creator(article['addedBy']['fullName'])
feed_entry.content(content=clean_content(article['body']['summary']),
type="html")
feed_entry.link(href="/".join((HOST, "community",
article['type']['slug'],
article['slug'])),
rel='alternate')
def generate_feed(atom_file, rss_file):
""" generate ATOM/RSS feed """
feed = FeedGenerator()
feed.load_extension("dc", atom=False, rss=True)
feed.title("GNS3 Discussions")
feed.description("GNS3 Discussions")
feed.link(href=HOST+"/community/featured", rel='alternate')
feed.language('en')
sess = requests_session_retry()
add_articles(feed, sess)
md5_hash = hashlib.md5()
for feed_entry in feed.entry():
md5_hash.update(feed_entry.id().encode('utf-8'))
feed.id(HOST + '/feed/' + md5_hash.hexdigest())
try:
if atom_file:
feed_type = "ATOM"
atom_file = os.path.expanduser(atom_file)
feed.atom_file(atom_file + ".new")
os.replace(atom_file + ".new", atom_file)
if rss_file:
feed_type = "RSS"
rss_file = os.path.expanduser(rss_file)
feed.rss_file(rss_file + ".new")
os.replace(rss_file + ".new", rss_file)
except OSError as err:
sys.exit("Can't create {} feed: {}".format(feed_type, err))
# Main
parser = argparse.ArgumentParser(add_help=False, \
description='%(prog)s - generate ATOM/RSS feed from GNS3 community website')
parser.add_argument('-h', '--help', '-?', action='help',
help='prints this screen')
parser.add_argument('-v', '--version', action='version',
version="%(prog)s v" + __version__,
help='prints version')
parser.add_argument('-a', '--atom', metavar="file",
help='generate ATOM feed')
parser.add_argument('-r', '--rss', metavar="file",
help='generate RSS feed')
args = parser.parse_args()
if not args.atom and not args.rss:
parser.error("at least one of the options -a/--atom or -r/--rss must be used.")
generate_feed(args.atom, args.rss)