So I went through the faq and generated a widget for statistics of the current torrents. I also sorted through what I will need to do the flotwidget graph over time but I need to sort out where the data will be stored and what will be fetching it first.
Patch is attached. It includes a generic json connector for getting random json data and then doing _something_ with it. There are a couple of fixmes in the code about whitelisting the urls the connector is allowed to visit.
A couple of questions: - why the @classmethod decorator use in the connectors? It doesn't SEEM to be necessary but I'm not sure if it was something I missed in my own testing - Related - what is the point of the registering in the IQuery/ICall/IConnect objects? Feels like a lot of redundant typing.
- It would make life somewhat easier if there was a better way from within a page to know where within the code it lived. I was doing a lot of grepping to figure out where to look for other things. A trackback when in development mode would help make finding things simpler.
Comments on the attached code welcome.
-sv
After a little bit of tweaking I was able to get the torrent statistics working, and have merged it into the master branch, along with the rest of the 'stats' branch (containing wiki & account stats).
luke
On Fri, Oct 23, 2009 at 03:53:43PM -0400, Seth Vidal wrote:
So I went through the faq and generated a widget for statistics of the current torrents. I also sorted through what I will need to do the flotwidget graph over time but I need to sort out where the data will be stored and what will be fetching it first.
Patch is attached. It includes a generic json connector for getting random json data and then doing _something_ with it. There are a couple of fixmes in the code about whitelisting the urls the connector is allowed to visit.
A couple of questions:
- why the @classmethod decorator use in the connectors? It doesn't
SEEM to be necessary but I'm not sure if it was something I missed in my own testing
- Related - what is the point of the registering in the
IQuery/ICall/IConnect objects? Feels like a lot of redundant typing.
- It would make life somewhat easier if there was a better way from
within a page to know where within the code it lived. I was doing a lot of grepping to figure out where to look for other things. A trackback when in development mode would help make finding things simpler.
Comments on the attached code welcome.
-sv
diff --git a/fedoracommunity/connectors/__init__.py b/fedoracommunity/connectors/__init__.py index d0f42db..61d5492 100644 --- a/fedoracommunity/connectors/__init__.py +++ b/fedoracommunity/connectors/__init__.py @@ -6,3 +6,5 @@ from fasconnector import FasConnector from bugzillaconnector import BugzillaConnector from planet import PlanetConnector from wikiconnector import WikiConnector +from torrentconnector import TorrentConnector +from jsonconnector import SimpleJsonConnector diff --git a/fedoracommunity/connectors/jsonconnector.py b/fedoracommunity/connectors/jsonconnector.py new file mode 100644 index 0000000..961ea92 --- /dev/null +++ b/fedoracommunity/connectors/jsonconnector.py @@ -0,0 +1,58 @@ +# This file is part of Fedora Community. +# Copyright (C) 2008-2009 Red Hat, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see http://www.gnu.org/licenses/.
+""" +:mod:`fedoracommunity.connectors.jsonconnector` - Simple Json Connector +=======================================================================
+This Connector works with any url which returns valid simplejson data
+.. moduleauthor:: Seth Vidal skvidal@fedoraproject.org +""" +import logging +log = logging.getLogger(__name__)
+from datetime import datetime, timedelta +from pylons import cache +from urllib import urlopen +import simplejson +from moksha.connector import IConnector, ICall, IQuery, ParamFilter
+class SimpleJsonConnector(IConnector, ICall, IQuery):
- _method_paths = {}
- _query_paths = {}
- def __init__(self, environ=None, request=None):
super(SimpleJsonConnector, self).__init__(environ, request)
# FIXME - sanity check this url or run it past a whitelist or what not
- def call(self, url):
log.info('JsonConnector.call(%s)' % url)
self._url = url
json_cache = cache.get_cache('json')
return json_cache.get_value(key=url,
createfunc=self._get_json_url,
expiretime=1800)
- def _get_json_url(self):
# FIXME - LOTS OF ERROR CHECKING PLEASE
# grab the json_url
json_fp = urlopen(self._url)
# decode it into python using simplejson
json_data = simplejson.load(json_fp)
json_fp.close()
# return the object you get from it
return json_data
diff --git a/fedoracommunity/connectors/torrentconnector.py b/fedoracommunity/connectors/torrentconnector.py new file mode 100644 index 0000000..17e4300 --- /dev/null +++ b/fedoracommunity/connectors/torrentconnector.py @@ -0,0 +1,157 @@ +# This file is part of Fedora Community. +# Copyright (C) 2008-2009 Red Hat, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see http://www.gnu.org/licenses/.
+""" +:mod:`fedoracommunity.connectors.torrentconnector` - Fedora bittorrent connector +=======================================================================
+This Connector works with the jsonconnector
+.. moduleauthor:: Seth Vidal skvidal@fedoraproject.org +"""
+from datetime import datetime, timedelta +from pylons import cache, config +from moksha.lib.helpers import defaultdict +from jsonconnector import SimpleJsonConnector +from operator import itemgetter +import logging +log = logging.getLogger(__name__)
+class TorrentConnector(SimpleJsonConnector):
- _method_paths = {}
- _query_paths = {}
- def __init__(self, environ=None, request=None):
log.info('Torrent Connector initialized(%s)' % locals())
super(TorrentConnector, self).__init__(environ, request)
- @classmethod
- def register(cls):
cls.stats_url = config.get('fedoracommunity.connector.torrent.statsurl',
'http://torrent.fedoraproject.org/stats/current-stats.json')
cls.register_query_most_active_torrents()
cls.register_query_most_downloaded_torrents()
- @classmethod
- def register_query_most_active_torrents(cls):
path = cls.register_query(
'query_most_active_torrents',
cls.query_most_active_torrents,
primary_key_col = 'torrent_name',
default_sort_col = 'number_of_downloaders',
default_sort_order = -1,
can_paginate = True)
path.register_column('torrent_name',
default_visible = True,
can_sort = False,
can_filter_wildcards = False)
path.register_column('number_of_downloaders',
default_visible = True,
can_sort = False,
can_filter_wildcards = False)
- def query_most_active_torrents(self, start_row=0,
rows_per_page=10,
order=1,
sort_col=None,
filters=None,
**params):
torrents = self.call(self.stats_url)
most_downloaders = sorted(torrents,key=itemgetter('downloaders'),
reverse=True)
results = []
for torrent in most_downloaders:
results.append({'number_of_downloaders': torrent['downloaders'],
'torrent_name': torrent['name']})
return (len(results), results[start_row:start_row+rows_per_page])
- @classmethod
- def register_query_most_downloaded_torrents(cls):
path = cls.register_query(
'query_most_downloaded_torrents',
cls.query_most_downloaded_torrents,
primary_key_col = 'torrent_name',
default_sort_col = 'number_of_completed',
default_sort_order = -1,
can_paginate = True)
path.register_column('torrent_name',
default_visible = True,
can_sort = False,
can_filter_wildcards = False)
path.register_column('number_of_completed',
default_visible = True,
can_sort = False,
can_filter_wildcards = False)
- def query_most_downloaded_torrents(self, start_row=0,
rows_per_page=10,
order=1,
sort_col=None,
filters=None,
**params):
log.info('most_downloaded_torrents called(%s)' % locals())
torrents = self.call(self.stats_url)
most_downloaded = sorted(torrents,key=itemgetter('completed'),
reverse=True)
results = []
for torrent in most_downloaded:
results.append({'number_of_completed': torrent['completed'],
'torrent_name': torrent['name']})
return (len(results), results[start_row:start_row+rows_per_page])
- def query_most_active_torrents_history(self, torrent_count=10, **params):
# obviously this is garbage data - just playing with flotwidget
# line graphs
flot_data = {}
flot_data['data'] = []
flot_data['options'] = {}
flot_data['options']['legend'] = { 'show': True,
'noColumns': '1',}
flot_data['data'].append({
# data sets should be [timestamp, downloaders]
'data': [[1, 1], [2,2], [3,3], [4,5], [5, 6]],
'lines': {'show': True},
'points': {'show': True},
'label': 'Torrent1',
})
flot_data['data'].append({
'data': [[2, 10], [3,23], [4,10], [5,8]],
'lines': {'show': True},
'points': {'show': True},
'label': 'Torrent2',
})
return flot_data
diff --git a/fedoracommunity/mokshaapps/statistics/controllers/root.py b/fedoracommunity/mokshaapps/statistics/controllers/root.py index ae0755e..ca4c440 100644 --- a/fedoracommunity/mokshaapps/statistics/controllers/root.py +++ b/fedoracommunity/mokshaapps/statistics/controllers/root.py @@ -21,6 +21,7 @@ from moksha.api.widgets.containers.dashboardcontainer import applist_widget
from fedoracommunity.widgets import SubTabbedContainer from fedoracommunity.mokshaapps.statistics.widgets import wiki_stats_dashboard +from fedoracommunity.mokshaapps.statistics.widgets import torrent_stats_dashboard #from fedoracommunity.mokshaapps.statistics.widgets import updates_stats_dashboard
class StatsNavContainer(SubTabbedContainer): @@ -30,11 +31,13 @@ class StatsNavContainer(SubTabbedContainer): tabs = ( Category('Applications', ( MokshaApp('Wiki', 'fedoracommunity.statistics/wiki', params={}),
MokshaApp('Torrents', 'fedoracommunity.statistics/torrents', params={}), #MokshaApp('Updates', 'fedoracommunity.statistics/updates', params={}), ), ),
)
statistics_nav_container = StatsNavContainer('statistics_nav_container')
@@ -50,6 +53,11 @@ class RootController(Controller): tmpl_context.widget = wiki_stats_dashboard return dict(options={})
- @expose('mako:moksha.templates.widget')
- def torrents(self):
tmpl_context.widget = torrent_stats_dashboard
return dict(options={})
- #@expose('mako:moksha.templates.widget') #def updates(self): # tmpl_context.widget = updates_stats_dashboard
diff --git a/fedoracommunity/mokshaapps/statistics/templates/most_active_torrents.mak b/fedoracommunity/mokshaapps/statistics/templates/most_active_torrents.mak new file mode 100644 index 0000000..1244955 --- /dev/null +++ b/fedoracommunity/mokshaapps/statistics/templates/most_active_torrents.mak @@ -0,0 +1,55 @@ +<div class="list header-list">
<script type="text/javascript">
function _text_filter(text) {
var results = $("<div />");
var ul = $("<ul />");
results.append(ul);
var v=text.split('\n');
for (i in v) {
line = v[i];
ul.append("<li>" + line + "</li>");
}
return results.html();
}
- </script>
<table id="${id}">
<thead>
<th>Torrent Name</th>
<th>Number of downloaders</th>
</thead>
<tbody class="rowtemplate">
<tr>
<td>
@{torrent_name}
</td>
<td>
@{number_of_downloaders}
</td>
</tr>
</tbody>
</table>
+<div id="grid-controls" if="total_rows == 0">
<div class="message template" id="info_display" >
No torrents being downloaded.
</div>
+</div> +<div id="grid-controls" if="visible_rows >= total_rows && total_rows != 0">
<div class="message template" id="info_display" >
Viewing all torrents being downloaded.
</div>
<div class="pager template" id="pager" type="more_link">
<a href="http://torrent.fedoraproject.org:6969/">View complete torrent statistics information ></a>
</div>
+</div> +<div id="grid-controls" if="visible_rows < total_rows">
<div class="message template" id="info_display" >
Viewing @{first_visible_row}-@{last_visible_row} of @{total_rows} most active torrents
</div>
<div class="pager" id="pager" type="numeric" ></div>
<div class="pager template" id="pager" type="more_link">
<a href="@{more_link}" moksha_url="dynamic">View more torrents ></a>
</div>
+</div> +</div> diff --git a/fedoracommunity/mokshaapps/statistics/templates/most_downloaded_torrents.mak b/fedoracommunity/mokshaapps/statistics/templates/most_downloaded_torrents.mak new file mode 100644 index 0000000..b3d6d22 --- /dev/null +++ b/fedoracommunity/mokshaapps/statistics/templates/most_downloaded_torrents.mak @@ -0,0 +1,55 @@ +<div class="list header-list">
<script type="text/javascript">
function _text_filter(text) {
var results = $("<div />");
var ul = $("<ul />");
results.append(ul);
var v=text.split('\n');
for (i in v) {
line = v[i];
ul.append("<li>" + line + "</li>");
}
return results.html();
}
- </script>
<table id="${id}">
<thead>
<th>Torrent Name</th>
<th>Number of completed downloads</th>
</thead>
<tbody class="rowtemplate">
<tr>
<td>
@{torrent_name}
</td>
<td>
@{number_of_completed}
</td>
</tr>
</tbody>
</table>
+<div id="grid-controls" if="total_rows == 0">
<div class="message template" id="info_display" >
No torrents downloaded.
</div>
+</div> +<div id="grid-controls" if="visible_rows >= total_rows && total_rows != 0">
<div class="message template" id="info_display" >
Viewing all torrents being downloaded.
</div>
<div class="pager template" id="pager" type="more_link">
<a href="http://torrent.fedoraproject.org:6969/">View complete torrent statistics information ></a>
</div>
+</div> +<div id="grid-controls" if="visible_rows < total_rows">
<div class="message template" id="info_display" >
Viewing @{first_visible_row}-@{last_visible_row} of @{total_rows} most downloaded torrents
</div>
<div class="pager" id="pager" type="numeric" ></div>
<div class="pager template" id="pager" type="more_link">
<a href="@{more_link}" moksha_url="dynamic">View more torrents ></a>
</div>
+</div> +</div> diff --git a/fedoracommunity/mokshaapps/statistics/widgets/__init__.py b/fedoracommunity/mokshaapps/statistics/widgets/__init__.py index 3709e0d..a278563 100644 --- a/fedoracommunity/mokshaapps/statistics/widgets/__init__.py +++ b/fedoracommunity/mokshaapps/statistics/widgets/__init__.py @@ -1 +1,2 @@ from wiki import * +from torrent import * diff --git a/fedoracommunity/mokshaapps/statistics/widgets/torrent.py b/fedoracommunity/mokshaapps/statistics/widgets/torrent.py new file mode 100644 index 0000000..00493ba --- /dev/null +++ b/fedoracommunity/mokshaapps/statistics/widgets/torrent.py @@ -0,0 +1,70 @@ +# This file is part of Fedora Community. +# Copyright (C) 2008-2009 Red Hat, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see http://www.gnu.org/licenses/.
+from pylons import cache +from datetime import datetime, timedelta +from moksha.api.widgets import Grid +from moksha.api.widgets.containers import DashboardContainer +from moksha.lib.helpers import Category, Widget as MokshaWidget, defaultdict +from moksha.api.connectors import get_connector +from fedoracommunity.widgets.flot import FlotWidget
+class MostActiveTorrents(Grid):
- template = 'mako:fedoracommunity.mokshaapps.statistics.templates.most_active_torrents'
- resource = 'torrent'
- resource_path='query_most_active_torrents'
- numericPager = True
+most_active_torrent = MostActiveTorrents('most_active_torrents')
+class MostDownloadedTorrents(Grid):
- template = 'mako:fedoracommunity.mokshaapps.statistics.templates.most_downloaded_torrents'
- resource = 'torrent'
- resource_path='query_most_downloaded_torrents'
- numericPager = True
+most_downloaded_torrent = MostDownloadedTorrents('most_downloaded_torrents')
+class MostActiveTorrentsChart(FlotWidget):
- """show the history of the most active torrents and their activity
for however long we have in the backend"""
- def update_params(self, d):
torrent_connector = get_connector('torrent')
torrent_cache = cache.get_cache('torrent')
stats = torrent_cache.get_value(key='torrent_stats',
createfunc=torrent_connector.query_most_active_torrents_history,
expiretime=3600)
d.data = stats['data']
d.options = stats['options']
super(MostActiveTorrentsChart, self).update_params(d)
+most_active_torrents_chart = MostActiveTorrentsChart('most_active_torrents_chart')
+class TorrentStatisticsDashboard(DashboardContainer):
- layout = [
Category('left-content-column-apps', [
MokshaWidget('Most active torrent', most_active_torrent),
MokshaWidget('Most downloaded torrent', most_downloaded_torrent),
+# MokshaWidget('Most active torrents chart', most_active_torrents_chart),
]),
- ]
+torrent_stats_dashboard = TorrentStatisticsDashboard('torrent_stats') diff --git a/setup.py b/setup.py index 0834c0f..fb4d388 100755 --- a/setup.py +++ b/setup.py @@ -99,6 +99,7 @@ setup( planet = fedoracommunity.connectors:PlanetConnector yum = fedoracommunity.connectors:YumConnector wiki = fedoracommunity.connectors:WikiConnector
torrent = fedoracommunity.connectors:TorrentConnector
[moksha.application] login = fedoracommunity.mokshaapps.login:RootController
moksha mailing list moksha@lists.fedorahosted.org https://fedorahosted.org/mailman/listinfo/moksha