[copr] master: [frontend][backend] [rhbz:#1185959] - RFE: Present statistics about project popularity. A few more counters for downloads from backend's result directory. (69f6553)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 69f65530119bbc6577dd298c71f2ef164aa95f7a
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Wed Feb 25 18:17:49 2015 +0100
[frontend][backend] [rhbz:#1185959] - RFE: Present statistics about project popularity. A few more counters for downloads from backend's result directory.
>---------------------------------------------------------------
backend/conf/logstash/copr_backend.conf | 60 ++++++++++
backend/conf/logstash/lighttpd.pattern | 1 +
backend/copr-backend.spec | 12 ++
frontend/conf/logstash.conf | 37 ++++---
frontend/copr-frontend.spec | 5 +
frontend/coprs_frontend/coprs/__init__.py | 10 +-
frontend/coprs_frontend/coprs/helpers.py | 43 +++++++-
frontend/coprs_frontend/coprs/logic/stat_logic.py | 44 ++++++--
frontend/coprs_frontend/coprs/rmodels.py | 117 ++++++++++++++++++++
.../coprs/templates/coprs/detail/overview.html | 10 ++-
.../coprs/views/coprs_ns/coprs_general.py | 21 +++-
frontend/coprs_frontend/coprs/views/misc.py | 2 +-
.../coprs/views/stats_ns/stats_receiver.py | 21 +---
frontend/coprs_frontend/tests/test_rmodels.py | 53 +++++++++
14 files changed, 387 insertions(+), 49 deletions(-)
diff --git a/backend/conf/logstash/copr_backend.conf b/backend/conf/logstash/copr_backend.conf
new file mode 100644
index 0000000..e6a96e2
--- /dev/null
+++ b/backend/conf/logstash/copr_backend.conf
@@ -0,0 +1,60 @@
+input {
+ #file {
+ # path => "/var/log/copr/backend.log"
+ # type => "copr.backend.main"
+ #}
+ file {
+ path => "/var/log/lighttpd/access.log"
+ type => "lighttpd-access"
+ }
+}
+
+filter {
+ mutate {
+ add_tag => [ "backend" ]
+ }
+ if [type] == 'lighttpd-access' {
+ grok {
+ patterns_dir => "/usr/share/logstash/patterns"
+ pattern => "%{LIGHTTPD}"
+ }
+ date {
+ match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]
+ }
+ if "repodata/repomd.xml" in [request] and "devel/repodata/repomd.xml" not in [request] {
+ mutate { add_tag => "repomdxml" }
+ mutate { add_tag => "publish_stat" }
+ grok {
+ match => ["request", "/results/%{USERNAME:copr_user}/%{USERNAME:copr_project_name}/%{USERNAME:copr_chroot}/repodata/repomd.xml"]
+ }
+ }
+ if [request] =~ "rpm$" {
+ mutate { add_tag => "rpm" }
+ mutate { add_tag => "publish_stat" }
+ grok {
+ match => ["request", "/results/%{USERNAME:copr_user}/%{USERNAME:copr_project_name}/%{USERNAME:copr_chroot}/%{USERNAME:copr_build_dir}/%{USERNAME:copr_rpm}"]
+ }
+ }
+ }
+}
+
+output {
+ if "publish_stat" in [tags] {
+ http {
+ url => "http://copr-fe-dev.cloud.fedoraproject.org/stats_rcv/from_logstash"
+ format => "json"
+ http_method => "post"
+ }
+ }
+
+ file {
+ path => "/var/log/logstash/all.log"
+ codec => "rubydebug"
+ }
+
+ file {
+ path => "/tmp/logstashall.log"
+ codec => "rubydebug"
+ }
+
+}
diff --git a/backend/conf/logstash/lighttpd.pattern b/backend/conf/logstash/lighttpd.pattern
new file mode 100644
index 0000000..7dce2a0
--- /dev/null
+++ b/backend/conf/logstash/lighttpd.pattern
@@ -0,0 +1 @@
+LIGHTTPD %{IPORHOST:clientip} %{IPORHOST:httphost} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{URIPATHPARAM:request}(?: HTTP/%{NUMBER:httpversion})|-)" %{NUMBER:response} (?:%{NUMBER:bytes}|-) "(?:%{URI:referrer}|-)" %{QS:agent}
diff --git a/backend/copr-backend.spec b/backend/copr-backend.spec
index ec3dc85..bb39b63 100644
--- a/backend/copr-backend.spec
+++ b/backend/copr-backend.spec
@@ -72,6 +72,7 @@ Requires: fedmsg
Requires: gawk
Requires: crontabs
Requires: python-paramiko
+Requires: logstash
Requires(post): systemd
Requires(preun): systemd
@@ -145,6 +146,13 @@ touch %{buildroot}%{_var}/run/copr-backend/copr-be.pid
install -m 0644 copr-backend.service %{buildroot}/%{_unitdir}/
install -m 0644 conf/copr.sudoers.d %{buildroot}%{_sysconfdir}/sudoers.d/copr
+
+install -d %{buildroot}%{_sysconfdir}/logstash.d
+cp -a conf/logstash/copr_backend.conf %{buildroot}%{_sysconfdir}/logstash.d/copr_backend.conf
+install -d %{buildroot}%{_datadir}/logstash/patterns/
+cp -a conf/logstash/lighttpd.pattern %{buildroot}%{_datadir}/logstash/patterns/lighttpd.pattern
+
+
#doc
cp -a documentation/python-doc %{buildroot}%{_pkgdocdir}/
cp -a conf/playbooks %{buildroot}%{_pkgdocdir}/
@@ -163,6 +171,7 @@ useradd -r -g copr -G lighttpd -s /bin/bash -c "COPR user" copr
%post
%systemd_post copr-backend.service
+%systemd_post logstash.service
%preun
%systemd_preun copr-backend.service
@@ -197,6 +206,9 @@ useradd -r -g copr -G lighttpd -s /bin/bash -c "COPR user" copr
%{_bindir}/*
%config(noreplace) %{_sysconfdir}/cron.daily/copr-backend
+%config(noreplace) %{_sysconfdir}/logstash.d/copr_backend.conf
+%{_datadir}/logstash/patterns/lighttpd.pattern
+
%config(noreplace) %attr(0600, root, root) %{_sysconfdir}/sudoers.d/copr
diff --git a/frontend/conf/logstash.conf b/frontend/conf/logstash.conf
index 82aa01d..36cb8cd 100644
--- a/frontend/conf/logstash.conf
+++ b/frontend/conf/logstash.conf
@@ -10,32 +10,39 @@ input {
}
filter {
- grok {
- type => "httpd-access"
- pattern => "%{COMBINEDAPACHELOG}"
+ mutate { add_tag => "frontend" }
+ if [type] == "httpd-access" {
+ if "POST /stats_rcv/from_logstash HTTP" in [message] {
+ drop{}
+ }
+ grok {
+ type => "httpd-access"
+ pattern => "%{COMBINEDAPACHELOG}"
+ }
+
+ if [request] =~ ".repo$" {
+ mutate { add_tag => "repo_dl" }
+ grok {
+ match => ["request", "/coprs/%{USERNAME:copr_user}/%{USERNAME:copr_project_name}/repo/%{USERNAME:copr_name_release}/%{USERNAME:copr_repo_file}"]
+ }
+ }
+
}
}
output {
if [type] == "httpd-access" {
- if ".repo" in [request] and "stats_rcv" not in [request] {
+ if "repo_dl" in [tags] {
http {
url => "http://127.0.0.1/stats_rcv/from_logstash"
format => "json"
http_method => "post"
- message => "foobar"
}
}
- #file {
- # path => "/var/log/logstash/web.log"
- # codec => rubydebug {}
- #
- #}
}
- # file {
- # path => "/var/log/logstash/all.log"
- # codec => rubydebug {}
- # }
-
+ file {
+ path => "/var/log/logstash/all.log"
+ codec => rubydebug {}
+ }
}
diff --git a/frontend/copr-frontend.spec b/frontend/copr-frontend.spec
index bf83869..4439e30 100644
--- a/frontend/copr-frontend.spec
+++ b/frontend/copr-frontend.spec
@@ -59,6 +59,9 @@ Requires: python-mock
Requires: python-decorator
Requires: yum
Requires: logstash
+Requires: redis
+Requires: python-redis
+Requires: python-dateutil
%if 0%{?rhel} < 7 && 0%{?rhel} > 0
BuildRequires: python-argparse
%endif
@@ -72,6 +75,8 @@ BuildRequires: python-flask-whooshee
BuildRequires: python-pylibravatar
BuildRequires: python-flask-wtf
BuildRequires: python-netaddr
+BuildRequires: python-redis
+BuildRequires: python-dateutil
BuildRequires: pytest
BuildRequires: yum
BuildRequires: python-flexmock
diff --git a/frontend/coprs_frontend/coprs/__init__.py b/frontend/coprs_frontend/coprs/__init__.py
index 492dec3..e86bb64 100644
--- a/frontend/coprs_frontend/coprs/__init__.py
+++ b/frontend/coprs_frontend/coprs/__init__.py
@@ -3,9 +3,9 @@ from __future__ import with_statement
import os
import flask
-from flask.ext.sqlalchemy import SQLAlchemy
-from flask.ext.openid import OpenID
-from flask.ext.whooshee import Whooshee
+from flask_sqlalchemy import SQLAlchemy
+from flask_openid import OpenID
+from flask_whooshee import Whooshee
app = flask.Flask(__name__)
@@ -25,12 +25,16 @@ oid = OpenID(app, app.config["OPENID_STORE"], safe_roots=[])
db = SQLAlchemy(app)
whooshee = Whooshee(app)
+
import coprs.filters
import coprs.log
from coprs.log import setup_log
import coprs.models
import coprs.whoosheers
+from coprs.helpers import RedisConnectionProvider
+rcp = RedisConnectionProvider(config=app.config)
+
from coprs.views import admin_ns
from coprs.views.admin_ns import admin_general
from coprs.views import api_ns
diff --git a/frontend/coprs_frontend/coprs/helpers.py b/frontend/coprs_frontend/coprs/helpers.py
index f4f8efa..811a481 100644
--- a/frontend/coprs_frontend/coprs/helpers.py
+++ b/frontend/coprs_frontend/coprs/helpers.py
@@ -4,6 +4,10 @@ import string
import urlparse
import flask
+from dateutil import parser as dt_parser
+
+from redis import StrictRedis
+
from coprs import constants
from coprs import app
@@ -21,7 +25,10 @@ def generate_api_token(size=30):
return ''.join(random.choice(string.ascii_lowercase) for x in range(size))
-REPO_DL_STAT_FMT = "{user}@{copr}:{name_release}"
+REPO_DL_STAT_FMT = "repo_dl_stat::{copr_user}@{copr_project_name}:{copr_name_release}"
+CHROOT_REPO_MD_DL_STAT_FMT = "chroot_repo_metadata_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}"
+CHROOT_RPMS_DL_STAT_FMT = "chroot_rpms_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}"
+PROJECT_RPMS_DL_STAT_FMT = "project_rpms_dl_stat:hset::{copr_user}@{copr_project_name}"
class CounterStatType(object):
@@ -268,3 +275,37 @@ class Serializer(object):
@property
def serializable_attributes(self):
return map(lambda x: x.name, self.__table__.columns)
+
+
+class RedisConnectionProvider(object):
+ def __init__(self, config):
+ self.host = config.get("redis_host", "127.0.0.1")
+ self.port = int(config.get("redis_port", "6379"))
+
+ def get_connection(self):
+ return StrictRedis(host=self.host, port=self.port)
+
+
+def get_redis_connection():
+ """
+ Creates connection to redis, now we use default instance at localhost, no config needed
+ """
+ return StrictRedis()
+
+
+def dt_to_unixtime(dt):
+ """
+ Converts datetime to unixtime
+ :param dt: DateTime instance
+ :rtype: float
+ """
+ return float(dt.strftime('%s'))
+
+
+def string_dt_to_unixtime(dt_string):
+ """
+ Converts datetime to unixtime from string
+ :param dt_string: datetime string
+ :rtype: str
+ """
+ return dt_to_unixtime(dt_parser.parse(dt_string))
diff --git a/frontend/coprs_frontend/coprs/logic/stat_logic.py b/frontend/coprs_frontend/coprs/logic/stat_logic.py
index c1b2e0f..a2e4062 100644
--- a/frontend/coprs_frontend/coprs/logic/stat_logic.py
+++ b/frontend/coprs_frontend/coprs/logic/stat_logic.py
@@ -3,6 +3,8 @@ import json
import os
import pprint
import time
+
+
from sqlalchemy import or_
from sqlalchemy import and_
@@ -13,11 +15,11 @@ from coprs import db
from coprs import exceptions
from coprs.models import CounterStat
from coprs import helpers
-from coprs.helpers import REPO_DL_STAT_FMT
+from coprs.helpers import REPO_DL_STAT_FMT, CHROOT_REPO_MD_DL_STAT_FMT, dt_to_unixtime, string_dt_to_unixtime, \
+ CHROOT_RPMS_DL_STAT_FMT, PROJECT_RPMS_DL_STAT_FMT
from coprs import signals
from coprs.helpers import CounterStatType
-
-
+from coprs.rmodels import TimedStatEvents
class CounterStatLogic(object):
@@ -65,9 +67,9 @@ class CounterStatLogic(object):
chroot_by_stat_name = {}
for chroot in copr.active_chroots:
kwargs = {
- "user": copr.owner.name,
- "copr": copr.name,
- "name_release": chroot.name_release
+ "copr_user": copr.owner.name,
+ "copr_project_name": copr.name,
+ "copr_name_release": chroot.name_release
}
chroot_by_stat_name[REPO_DL_STAT_FMT.format(**kwargs)] = chroot.name_release
@@ -83,6 +85,30 @@ class CounterStatLogic(object):
return repo_dl_stats
-
-
-
+def handle_logstash(rc, ls_data):
+ """
+ :param rc: connection to redis
+ :type rc: StrictRedis
+
+ :param ls_data: log stash record
+ :type ls_data: dict
+ """
+ dt_unixtime = string_dt_to_unixtime(ls_data["@timestamp"])
+ app.logger.debug("got ls_data: {}".format(ls_data))
+
+ if "tags" in ls_data:
+ tags = set(ls_data["tags"])
+ if "frontend" in tags and "repo_dl":
+ name = REPO_DL_STAT_FMT.format(**ls_data)
+ CounterStatLogic.incr(name=name, counter_type=CounterStatType.REPO_DL)
+ db.session.commit()
+
+ if "backend" in tags and "repomdxml" in tags:
+ key = CHROOT_REPO_MD_DL_STAT_FMT.format(**ls_data)
+ TimedStatEvents.add_event(rc, key, timestamp=dt_unixtime)
+
+ if "backend" in tags and "rpm" in tags:
+ key_chroot = CHROOT_RPMS_DL_STAT_FMT.format(**ls_data)
+ key_project = PROJECT_RPMS_DL_STAT_FMT.format(**ls_data)
+ TimedStatEvents.add_event(rc, key_chroot, timestamp=dt_unixtime)
+ TimedStatEvents.add_event(rc, key_project, timestamp=dt_unixtime)
diff --git a/frontend/coprs_frontend/coprs/rmodels.py b/frontend/coprs_frontend/coprs/rmodels.py
new file mode 100644
index 0000000..43734ae
--- /dev/null
+++ b/frontend/coprs_frontend/coprs/rmodels.py
@@ -0,0 +1,117 @@
+# coding: utf-8
+
+""" Models to redis entities """
+import time
+from math import ceil
+from datetime import datetime, timedelta
+from redis import StrictRedis
+
+
+class GenericRedisModel(object):
+ _KEY_BASE = "copr:generic"
+
+ @classmethod
+ def _get_key(cls, name, prefix=None):
+ if prefix:
+ return "{}:{}:{}".format(prefix, cls._KEY_BASE, name)
+ else:
+ return "{}:{}".format(cls._KEY_BASE, name)
+
+
+class TimedStatEvents(GenericRedisModel):
+ """
+ Wraps hset structure, where:
+ **key** - name of event, fix prefix specifying events type
+ **member** - bucket representing one day
+ **score** - events count
+ """
+ _KEY_BASE = "copr:tse"
+
+ @staticmethod
+ def timestamp_to_day(ut):
+ """
+ :param ut: unix timestamp
+ :type ut: float
+ :return: name for the day bucket
+ """
+ td = timedelta(days=1).total_seconds()
+ return int(ceil(ut / td))
+
+ @classmethod
+ def gen_days_interval(cls, min_ts, max_ts):
+ """
+ Generate list of days bucket names which contains
+ all events between `min_ts` and `max_ts`
+ :param min_ts: min unix timestamp
+ :param max_ts: max unix timestamp
+ :rtype: list
+ """
+ start_ut = cls.timestamp_to_day(min_ts)
+ end_ut = cls.timestamp_to_day(max_ts)
+
+ return range(start_ut, end_ut + 1)
+
+ @classmethod
+ def add_event(cls, rconnect, name, timestamp, count=1, prefix=None):
+ """
+ Stoted new event to redist
+ :param rconnect: Connection to a redis
+ :type rconnect: StrictRedis
+ :param name: statistics name
+ :param timestamp: timestamp of event
+ :param count: number of events, default=1
+ :param prefix: prefix for statistics, default is None
+ """
+ count = int(count)
+ ut_day = cls.timestamp_to_day(timestamp)
+
+ key = cls._get_key(name, prefix)
+
+ rconnect.hincrby(key, ut_day, count)
+
+ @classmethod
+ def get_count(cls, rconnect, name, day_min=None, prefix=None, day_max=None):
+ """
+ Count total event occurency between day_min and day_max
+ :param rconnect: Connection to a redis
+ :type rconnect: StrictRedis
+ :param name: statistics name
+ :param day_min: default: seven days ago
+ :param day_max: default: tomorrow
+ :param prefix: prefix for statistics, default is None
+
+ :rtype: int
+ """
+ key = cls._get_key(name, prefix)
+ if day_min is None:
+ day_min = time.time() - timedelta(days=7).total_seconds()
+
+ if day_max is None:
+ day_max = time.time() + timedelta(days=1).total_seconds()
+
+ interval = cls.gen_days_interval(day_min, day_max)
+ if len(interval) == 0:
+ return 0
+
+ res = rconnect.hmget(key, interval)
+ return sum(int(amount) for amount in res if amount is not None)
+
+
+ @classmethod
+ def trim_before(cls, rconnect, name, threshold_timestamp,
+ prefix=None):
+ """
+ Removes all records occured before `threshold_timestamp`
+ :param rconnect: StrictRedis
+ :param name: statistics name
+ :param threshold_timestamp: int
+ :param prefix: prefix for statistics, default is None
+ """
+
+ key = cls._get_key(name, prefix)
+
+ threshold_day = cls.timestamp_to_day(threshold_timestamp) + 1
+ all_members = rconnect.hgetall(key)
+ to_del = [mb for mb in all_members.keys() if int(mb) < threshold_day]
+
+ rconnect.hdel(key, *to_del)
diff --git a/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html b/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html
index 5c9629a..96c76e6 100644
--- a/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html
+++ b/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html
@@ -35,7 +35,15 @@
{{ repo.arch_list|join(", ") }}
</td>
<td>
- <center> {{ repo.dl_stat }} </center>
+ <center> {{ repo.dl_stat }} </center> <br />
+ <!-- --->
+ # of repo dl last week<br />
+ {% for arch in repo.arch_list %}
+
+ {{arch}}: {{ repo.rpm_dl_stat[arch] }} <br />
+
+ {% endfor %}
+ <!-- -->
</td>
<td class="rightmost">
<a href="{{
diff --git a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py
index de05b68..2eb7312 100644
--- a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py
+++ b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py
@@ -12,18 +12,20 @@ from itertools import groupby
from coprs import app
from coprs import db
+from coprs import rcp
from coprs import exceptions
from coprs import forms
from coprs import helpers
from coprs import models
from coprs.logic.stat_logic import CounterStatLogic
+from coprs.rmodels import TimedStatEvents
from coprs.views.misc import login_required, page_not_found
from coprs.views.coprs_ns import coprs_ns
from coprs.logic import builds_logic, coprs_logic, actions_logic
-from coprs.helpers import parse_package_name, generate_repo_url
+from coprs.helpers import parse_package_name, generate_repo_url, CHROOT_RPMS_DL_STAT_FMT, CHROOT_REPO_MD_DL_STAT_FMT
@coprs_ns.route("/", defaults={"page": 1})
@@ -189,16 +191,31 @@ def copr_detail(username, coprname):
repos_info = {}
for chroot in copr.active_chroots:
+ chroot_rpms_dl_stat_key = CHROOT_REPO_MD_DL_STAT_FMT.format(
+ copr_user=copr.owner.name,
+ copr_project_name=copr.name,
+ copr_chroot=chroot.name,
+ )
+ chroot_rpms_dl_stat = TimedStatEvents.get_count(
+ rconnect=rcp.get_connection(),
+ name=chroot_rpms_dl_stat_key,
+ )
+
if chroot.name_release not in repos_info:
repos_info[chroot.name_release] = {
"name_release": chroot.name_release,
"name_release_human": chroot.name_release_human,
"arch_list": [chroot.arch],
"repo_file": "{}-{}-{}.repo".format(copr.owner.name, copr.name, chroot.name_release),
- "dl_stat": repo_dl_stat[chroot.name_release]
+ "dl_stat": repo_dl_stat[chroot.name_release],
+ "rpm_dl_stat": {
+ chroot.arch: chroot_rpms_dl_stat
+ }
}
else:
repos_info[chroot.name_release]["arch_list"].append(chroot.arch)
+ repos_info[chroot.name_release]["rpm_dl_stat"][chroot.arch] = chroot_rpms_dl_stat
+
repos_info_list = sorted(repos_info.values(), key=lambda rec: rec["name_release"])
return flask.render_template("coprs/detail/overview.html",
diff --git a/frontend/coprs_frontend/coprs/views/misc.py b/frontend/coprs_frontend/coprs/views/misc.py
index 6714f78..fd90267 100644
--- a/frontend/coprs_frontend/coprs/views/misc.py
+++ b/frontend/coprs_frontend/coprs/views/misc.py
@@ -261,7 +261,7 @@ def intranet_required(f):
def decorated_function(*args, **kwargs):
ip_addr = IPAddress(flask.request.remote_addr)
if not any(ip_addr in IPNetwork(addr_or_net)
- for addr_or_net in app.config.get(["INTRANET_IPS"], "127.0.0.1")):
+ for addr_or_net in app.config.get("INTRANET_IPS", ["127.0.0.1",])):
return ("Stats can be update only from intranet hosts, "
"not {}, check config\n".format(flask.request.remote_addr)), 403
diff --git a/frontend/coprs_frontend/coprs/views/stats_ns/stats_receiver.py b/frontend/coprs_frontend/coprs/views/stats_ns/stats_receiver.py
index 9b40e29..d1f5f15 100644
--- a/frontend/coprs_frontend/coprs/views/stats_ns/stats_receiver.py
+++ b/frontend/coprs_frontend/coprs/views/stats_ns/stats_receiver.py
@@ -1,12 +1,14 @@
# coding: utf-8
import flask
+from coprs import rcp
from coprs import app
from coprs import db
from coprs.helpers import REPO_DL_STAT_FMT, CounterStatType
from ..misc import intranet_required
from . import stats_rcv_ns
-from ...logic.stat_logic import CounterStatLogic
+from ...logic.stat_logic import CounterStatLogic, handle_logstash
+
@stats_rcv_ns.route("/")
def ping():
@@ -27,22 +29,7 @@ def increment(counter_type, name):
@intranet_required
def logstash_handler():
try:
- json = flask.request.json
- if "request" in json:
- # 0 1 2 3 4 5
- # "request": "/coprs/bob/foox/repo/epel-5/bob-foox-epel-5.repo",
- req_split = json["request"].split("/")
- kwargs = dict(
- user=req_split[2],
- copr=req_split[3],
- name_release=req_split[5]
- )
- name = REPO_DL_STAT_FMT.format(**kwargs)
- app.logger.debug("kwargs: {}; name: {}".format(kwargs, name))
-
- CounterStatLogic.incr(name=name,
- counter_type=CounterStatType.REPO_DL)
- db.session.commit()
+ handle_logstash(rcp.get_connection(), flask.request.json)
except Exception as err:
app.logger.exception(err)
diff --git a/frontend/coprs_frontend/tests/test_rmodels.py b/frontend/coprs_frontend/tests/test_rmodels.py
new file mode 100644
index 0000000..939426c
--- /dev/null
+++ b/frontend/coprs_frontend/tests/test_rmodels.py
@@ -0,0 +1,53 @@
+# coding: utf-8
+
+import time
+import pytest
+
+from redis import StrictRedis
+
+from coprs.rmodels import TimedStatEvents
+
+
+class TestRModels(object):
+
+ def setup_method(self, method):
+ self.rc = StrictRedis()
+ self.prefix = "copr:test:r_models"
+
+ self.time_now = time.time()
+
+ def teardown_method(self, method):
+ keys = self.rc.keys('{}*'.format(self.prefix))
+ if keys:
+ self.rc.delete(*keys)
+
+ def test_nop(self):
+ pass
+
+ def test_timed_stats_events(self):
+ TimedStatEvents.add_event(self.rc, name="foobar", prefix=self.prefix,
+ timestamp=self.time_now, )
+
+ assert TimedStatEvents.get_count(self.rc, name="foobar", prefix=self.prefix,) == 1
+ TimedStatEvents.add_event(self.rc, name="foobar", prefix=self.prefix,
+ timestamp=self.time_now, count=2)
+
+ assert TimedStatEvents.get_count(self.rc, name="foobar", prefix=self.prefix,) == 3
+
+ TimedStatEvents.add_event(self.rc, name="foobar", prefix=self.prefix,
+ timestamp=self.time_now - 1000000, count=2)
+ TimedStatEvents.add_event(self.rc, name="foobar", prefix=self.prefix,
+ timestamp=self.time_now - 3000000, count=3)
+
+ assert TimedStatEvents.get_count(self.rc, name="foobar", prefix=self.prefix,) == 3
+ assert TimedStatEvents.get_count(self.rc, name="foobar", prefix=self.prefix,
+ day_min=self.time_now - 2000000) == 5
+ assert TimedStatEvents.get_count(self.rc, name="foobar", prefix=self.prefix,
+ day_min=self.time_now - 5000000) == 8
+
+ TimedStatEvents.trim_before(self.rc, name="foobar",
+ prefix=self.prefix, threshold_timestamp=self.time_now - 200000)
+
+ assert TimedStatEvents.get_count(self.rc, name="foobar", prefix=self.prefix,
+ day_min=self.time_now - 5000000) == 3
+
9 years, 1 month
[copr] master: [keygen] fix SELinux context for /var/log/copr-keygen/ to be accessable by httpd process (2b6fd88)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 2b6fd882a89dc359aeda34915cdd73b454c42b9c
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Mon Feb 23 14:19:56 2015 +0100
[keygen] fix SELinux context for /var/log/copr-keygen/ to be accessable by httpd process
>---------------------------------------------------------------
keygen/copr-keygen.spec | 3 +++
1 files changed, 3 insertions(+), 0 deletions(-)
diff --git a/keygen/copr-keygen.spec b/keygen/copr-keygen.spec
index 471231a..a3fcef2 100644
--- a/keygen/copr-keygen.spec
+++ b/keygen/copr-keygen.spec
@@ -150,6 +150,9 @@ getent passwd copr-signer >/dev/null || \
%post
+semanage fcontext -a -t httpd_log_t '/var/log/copr-keygen(/.*)?'
+restorecon -rv /var/log/copr-keygen/
+
service httpd condrestart
%postun
9 years, 1 month
[copr] master: [backend] fixing pip requirements (e258b1a)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit e258b1a9b04344ab03f2b6011fbc5b958516c55f
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Mon Feb 23 13:18:21 2015 +0100
[backend] fixing pip requirements
>---------------------------------------------------------------
backend/requirements.txt | 1 -
1 files changed, 0 insertions(+), 1 deletions(-)
diff --git a/backend/requirements.txt b/backend/requirements.txt
index 0d72690..2b6438f 100644
--- a/backend/requirements.txt
+++ b/backend/requirements.txt
@@ -1,7 +1,6 @@
setproctitle
PyYAML
# ansible
-setproctitle
redis
retask
python-daemon
9 years, 1 month
[copr] master: [backend] add pip requirement for: netaddr (1ca23b7)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 1ca23b75abdde8666066c8b7f550a141f126b29d
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Mon Feb 23 13:04:35 2015 +0100
[backend] add pip requirement for: netaddr
>---------------------------------------------------------------
backend/requirements.txt | 1 +
1 files changed, 1 insertions(+), 0 deletions(-)
diff --git a/backend/requirements.txt b/backend/requirements.txt
index 71e1cdf..0d72690 100644
--- a/backend/requirements.txt
+++ b/backend/requirements.txt
@@ -10,3 +10,4 @@ IPy
# documentation
sphinx
sphinx-argparse
+netaddr
9 years, 1 month
[copr] master: fixing test suite for jenkins (1be3ebc)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 1be3ebcf6fb5b83fae6134af6b3727865337593e
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Mon Feb 23 13:00:16 2015 +0100
fixing test suite for jenkins
>---------------------------------------------------------------
test_suite.sh | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/test_suite.sh b/test_suite.sh
index 63dcf2d..61112e9 100644
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -8,7 +8,7 @@ source _venv/bin/activate
# sphinx flask flask-script SQLAlchemy==0.8.7 flask-whooshee Flask-OpenID Flask-SQLAlchemy==1.0 Flask-WTF blinker pytz markdown pyLibravatar pydns flexmock whoosh decorator
-pip install pytest mock pytest-cov ipdb redis bunch PyYAML
+pip install pytest mock pytest-cov ipdb redis bunch PyYAML setproctitle ansible
cp -rv /usr/lib/python2.7/site-packages/rpmUtils _venv/lib/python2.7/site-packages/
9 years, 1 month
[copr] master: [frontend] safer /start_rcv/from_logstash handler (fedd2da)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit fedd2da3e6792b11b89f3f3598047feeb48c6e55
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Mon Feb 23 12:58:40 2015 +0100
[frontend] safer /start_rcv/from_logstash handler
>---------------------------------------------------------------
.../coprs/views/stats_ns/stats_receiver.py | 39 +++++++++++---------
1 files changed, 21 insertions(+), 18 deletions(-)
diff --git a/frontend/coprs_frontend/coprs/views/stats_ns/stats_receiver.py b/frontend/coprs_frontend/coprs/views/stats_ns/stats_receiver.py
index 3f811ba..9b40e29 100644
--- a/frontend/coprs_frontend/coprs/views/stats_ns/stats_receiver.py
+++ b/frontend/coprs_frontend/coprs/views/stats_ns/stats_receiver.py
@@ -22,25 +22,28 @@ def increment(counter_type, name):
db.session.commit()
return "", 201
+
@stats_rcv_ns.route("/from_logstash", methods=['POST'])
-@intranet_required # ?
+@intranet_required
def logstash_handler():
- # import ipdb; ipdb.set_trace()
- json = flask.request.json
- if "request" in json:
- # 0 1 2 3 4 5
- # "request": "/coprs/bob/foox/repo/epel-5/bob-foox-epel-5.repo",
- req_split = json["request"].split("/")
- kwargs = dict(
- user=req_split[2],
- copr=req_split[3],
- name_release=req_split[5]
- )
- name = REPO_DL_STAT_FMT.format(**kwargs)
- app.logger.debug("kwargs: {}; name: {}".format(kwargs, name))
-
- CounterStatLogic.incr(name=name,
- counter_type=CounterStatType.REPO_DL)
- db.session.commit()
+ try:
+ json = flask.request.json
+ if "request" in json:
+ # 0 1 2 3 4 5
+ # "request": "/coprs/bob/foox/repo/epel-5/bob-foox-epel-5.repo",
+ req_split = json["request"].split("/")
+ kwargs = dict(
+ user=req_split[2],
+ copr=req_split[3],
+ name_release=req_split[5]
+ )
+ name = REPO_DL_STAT_FMT.format(**kwargs)
+ app.logger.debug("kwargs: {}; name: {}".format(kwargs, name))
+
+ CounterStatLogic.incr(name=name,
+ counter_type=CounterStatType.REPO_DL)
+ db.session.commit()
+ except Exception as err:
+ app.logger.exception(err)
return "", 201
9 years, 1 month
[copr] master: [frontend] views.misc.intranet_required use netaddr lib to compare addresses (ef0f12a)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit ef0f12a1216d3749b42302efb6509e67b0052c7e
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Fri Feb 20 23:22:11 2015 +0100
[frontend] views.misc.intranet_required use netaddr lib to compare addresses
>---------------------------------------------------------------
frontend/copr-frontend.spec | 2 ++
frontend/coprs_frontend/config/copr_devel.conf | 4 +++-
frontend/coprs_frontend/coprs/views/misc.py | 14 ++++++--------
3 files changed, 11 insertions(+), 9 deletions(-)
diff --git a/frontend/copr-frontend.spec b/frontend/copr-frontend.spec
index 3df1de9..bf83869 100644
--- a/frontend/copr-frontend.spec
+++ b/frontend/copr-frontend.spec
@@ -51,6 +51,7 @@ Requires: python-pylibravatar
Requires: python-whoosh >= 2.5.3
Requires: pytz
Requires: python-six
+Requires: python-netaddr
# for tests:
Requires: pytest
Requires: python-flexmock
@@ -70,6 +71,7 @@ BuildRequires: python-flask-openid
BuildRequires: python-flask-whooshee
BuildRequires: python-pylibravatar
BuildRequires: python-flask-wtf
+BuildRequires: python-netaddr
BuildRequires: pytest
BuildRequires: yum
BuildRequires: python-flexmock
diff --git a/frontend/coprs_frontend/config/copr_devel.conf b/frontend/coprs_frontend/config/copr_devel.conf
index d6c5615..c586406 100644
--- a/frontend/coprs_frontend/config/copr_devel.conf
+++ b/frontend/coprs_frontend/config/copr_devel.conf
@@ -44,4 +44,6 @@ ENFORCE_PROTOCOL_FOR_FRONTEND_URL = "http"
PUBLIC_COPR_HOSTNAME = "copr-fe-dev.cloud.fedoraproject.org"
LOG_FILENAME="/tmp/copr_frontend.log"
-INTRANET_IPS = ["127.0.0.1", "192.168.1.102"]
+
+# IP or subnet
+INTRANET_IPS = ["127.0.0.1", "192.168.1.0/24"]
diff --git a/frontend/coprs_frontend/coprs/views/misc.py b/frontend/coprs_frontend/coprs/views/misc.py
index db29c29..6714f78 100644
--- a/frontend/coprs_frontend/coprs/views/misc.py
+++ b/frontend/coprs_frontend/coprs/views/misc.py
@@ -2,6 +2,7 @@ import base64
import datetime
import functools
+from netaddr import IPAddress, IPNetwork
import re
import flask
@@ -258,14 +259,11 @@ def backend_authenticated(f):
def intranet_required(f):
@functools.wraps(f)
def decorated_function(*args, **kwargs):
- # app.logger.debug("MY: {}".format(flask.request.remote_addr))
- # Use,
- # >>> import ipaddress
- # >>> ipaddress.ip_address('192.168.0.1') in ipaddress.ip_network('192.168.0.0/24')
- if not app.config["DEBUG"]:
- if flask.request.remote_addr not in app.config["INTRANET_IPS"]:
- return ("Stats can be update only from intranet hosts, "
- "not {}, check config\n".format(flask.request.remote_addr)), 403
+ ip_addr = IPAddress(flask.request.remote_addr)
+ if not any(ip_addr in IPNetwork(addr_or_net)
+ for addr_or_net in app.config.get(["INTRANET_IPS"], "127.0.0.1")):
+ return ("Stats can be update only from intranet hosts, "
+ "not {}, check config\n".format(flask.request.remote_addr)), 403
return f(*args, **kwargs)
return decorated_function
9 years, 1 month
[copr] master: [frontend] [rhbz:#1185959] RFE: Present statistics about project popularity: less logic in jinja template (64139b0)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 64139b0697134c8ce5f063d937d3f33fbdc3cdfe
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Fri Feb 20 14:40:12 2015 +0100
[frontend] [rhbz:#1185959] RFE: Present statistics about project popularity: less logic in jinja template
>---------------------------------------------------------------
frontend/coprs_frontend/coprs/config.py | 3 +-
frontend/coprs_frontend/coprs/log.py | 2 +-
frontend/coprs_frontend/coprs/models.py | 8 ++
.../coprs/templates/coprs/detail/overview.html | 75 +++++++-------------
.../coprs/views/coprs_ns/coprs_general.py | 32 ++++++--
.../coprs/views/stats_ns/stats_receiver.py | 2 +-
6 files changed, 62 insertions(+), 60 deletions(-)
diff --git a/frontend/coprs_frontend/coprs/config.py b/frontend/coprs_frontend/coprs/config.py
index b8bb37a..15f3c08 100644
--- a/frontend/coprs_frontend/coprs/config.py
+++ b/frontend/coprs_frontend/coprs/config.py
@@ -42,8 +42,9 @@ class Config(object):
# primary log file
LOG_FILENAME = "/var/log/copr/frontend.log"
- INTRANET_IPS = ["127.0.0.1"]
+ INTRANET_IPS = ["127.0.0.1"]
+ DEBUG = True
class ProductionConfig(Config):
DEBUG = False
diff --git a/frontend/coprs_frontend/coprs/log.py b/frontend/coprs_frontend/coprs/log.py
index 90d2022..8fd6d55 100644
--- a/frontend/coprs_frontend/coprs/log.py
+++ b/frontend/coprs_frontend/coprs/log.py
@@ -45,4 +45,4 @@ def setup_log():
handler.setLevel(log_level)
app.logger.addHandler(handler)
- app.logger.info("logging configuration finished")
+ app.logger.info("logging configuration finished, config: {}".format(app.config))
diff --git a/frontend/coprs_frontend/coprs/models.py b/frontend/coprs_frontend/coprs/models.py
index d6cfe5d..1e011c3 100644
--- a/frontend/coprs_frontend/coprs/models.py
+++ b/frontend/coprs_frontend/coprs/models.py
@@ -429,6 +429,14 @@ class MockChroot(db.Model, helpers.Serializer):
"""
return "{}-{}".format(self.os_release, self.os_version)
+
+ @property
+ def name_release_human(self):
+ """
+ Textual representation of name of this or release
+ """
+ return "{} {}".format(self.os_release, self.os_version)
+
@property
def os(self):
"""
diff --git a/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html b/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html
index 6063a33..5c9629a 100644
--- a/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html
+++ b/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html
@@ -18,66 +18,42 @@
</div>
<div class="dnf-enable-field"> # <input onClick="this.select();"type="text" value="dnf copr enable {{copr.owner.name}}/{{ copr.name}}" readonly="readonly">
More info about <a target="_blank" href="https://fedorahosted.org/copr/wiki/HowToEnableRepo">how to enable a repo on the wiki page.</a></div>
+
<table class="releases">
<tr>
<th class="leftmost">Release</th>
<th>Architecture</th>
-
- <th class="rightmost">Yum Repo <small>[Downloads]</small></th>
+ <th>#Downloads</th>
+ <th class="rightmost">Yum Repo </th>
</tr>
- <!-- TODO: remove complex logic from html, group by os-release in the python view -->
- {% for mock_chroot in copr.active_chroots %}
- {% if loop.index < copr.active_chroots|length %}
- {% if mock_chroot.os_release != copr.active_chroots[loop.index].os_release or
- mock_chroot.os_version != copr.active_chroots[loop.index].os_version %}
- {# next release is different => release-end #}
- <tr class="release-end">
- {% else %}
- <tr>
- {% endif %}
- {% else %}{# last line => release-end for sure #}
- <tr class="release-end">
- {% endif %}
- {% if mock_chroot.os_release != copr.active_chroots[loop.index0 - 1].os_release or
- mock_chroot.os_version != copr.active_chroots[loop.index0 - 1].os_version or
- loop.index0 == 0 %}
- {# previous os_release-os_version were different or this is the first one #}
- <td>{{ mock_chroot.os_release|capitalize }} {{ mock_chroot.os_version }}</td>
- {% else %}
- <td></td>
- {% endif %}
+ {% for repo in repos_info_list %}
+ <tr class="release-end">
+ <td class="leftmost">
+ {{ repo.name_release_human|capitalize }}
+ </td>
<td>
- {{ mock_chroot.arch }}
- {% if copr.buildroot_pkgs(mock_chroot): %}
- <a id="modified-chroot-{{mock_chroot.name}}">[modified]</a>
- {% endif %}
+ {{ repo.arch_list|join(", ") }}
</td>
<td>
- {% if mock_chroot.os_release != copr.active_chroots[loop.index0 - 1].os_release or
- mock_chroot.os_version != copr.active_chroots[loop.index0 - 1].os_version or
- loop.index0 == 0 %}
- {# previous os_release-os_version were different or this is the first one #}
-
- <a href="{{
- url_for(
- 'coprs_ns.generate_repo_file',
- username=copr.owner.name,
- coprname=copr.name,
- chroot=mock_chroot.os_release+"-"+mock_chroot.os_version,
- repofile=copr.owner.name+'-'+copr.name+'-'+mock_chroot.os_release+"-"+mock_chroot.os_version+'.repo',
- _external=True
- )|fix_url_https_frontend}}">
- {{ copr.owner.name }}-{{ copr.name }}-{{mock_chroot.os_release+"-"+mock_chroot.os_version}}.repo
- </a>
- <small>[ {{ repo_dl_stat[mock_chroot.name_release] }} ]</small>
- {% endif %}
+ <center> {{ repo.dl_stat }} </center>
</td>
-
- </tr>
- {% else %}
- <tr colspan="2"><td>No active releases</td></tr>
+ <td class="rightmost">
+ <a href="{{
+ url_for(
+ 'coprs_ns.generate_repo_file',
+ username=copr.owner.name,
+ coprname=copr.name,
+ name_release=repo.name_release,
+ repofile=repo.repo_file,
+ _external=True
+ )|fix_url_https_frontend}}">
+ {{ repo.repo_file }}
+ </a>
+ </td>
+ </tr>
{% endfor %}
</table>
+
{% if copr.repos_list %}
<h2>Repository List</h2>
<ul class=repos-list>
@@ -87,6 +63,7 @@
</ul>
{% endif %}
+
{% if g.user and g.user.can_edit(copr) and copr and copr.owner and not copr.auto_createrepo %}
<dt>
<!--<h2>Release options</h2>-->
diff --git a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py
index 75ade2f..de05b68 100644
--- a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py
+++ b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py
@@ -184,12 +184,28 @@ def copr_detail(username, coprname):
except sqlalchemy.orm.exc.NoResultFound:
return page_not_found(
"Copr with name {0} does not exist.".format(coprname))
- from collections import defaultdict
+
repo_dl_stat = CounterStatLogic.get_copr_repo_dl_stat(copr)
+
+ repos_info = {}
+ for chroot in copr.active_chroots:
+ if chroot.name_release not in repos_info:
+ repos_info[chroot.name_release] = {
+ "name_release": chroot.name_release,
+ "name_release_human": chroot.name_release_human,
+ "arch_list": [chroot.arch],
+ "repo_file": "{}-{}-{}.repo".format(copr.owner.name, copr.name, chroot.name_release),
+ "dl_stat": repo_dl_stat[chroot.name_release]
+ }
+ else:
+ repos_info[chroot.name_release]["arch_list"].append(chroot.arch)
+ repos_info_list = sorted(repos_info.values(), key=lambda rec: rec["name_release"])
+
return flask.render_template("coprs/detail/overview.html",
copr=copr,
form=form,
- repo_dl_stat=repo_dl_stat
+ repo_dl_stat=repo_dl_stat,
+ repos_info_list=repos_info_list,
)
@@ -509,9 +525,9 @@ def copr_legal_flag(username, coprname):
coprname=coprname))
-(a)coprs_ns.route("/<username>/<coprname>/repo/<chroot>/", defaults={"repofile": None})
-(a)coprs_ns.route("/<username>/<coprname>/repo/<chroot>/<repofile>")
-def generate_repo_file(username, coprname, chroot, repofile):
+(a)coprs_ns.route("/<username>/<coprname>/repo/<name_release>/", defaults={"repofile": None})
+(a)coprs_ns.route("/<username>/<coprname>/repo/<name_release>/<repofile>")
+def generate_repo_file(username, coprname, name_release, repofile):
""" Generate repo file for a given repo name.
Reponame = username-coprname """
# This solution is used because flask splits off the last part after a
@@ -520,7 +536,7 @@ def generate_repo_file(username, coprname, chroot, repofile):
reponame = "{0}-{1}".format(username, coprname)
- if repofile is not None and repofile != username + '-' + coprname + '-' + chroot + '.repo':
+ if repofile is not None and repofile != username + '-' + coprname + '-' + name_release + '.repo':
return page_not_found(
"Repository filename does not match expected: {0}"
.format(repofile))
@@ -534,9 +550,9 @@ def generate_repo_file(username, coprname, chroot, repofile):
return page_not_found(
"Project {0}/{1} does not exist".format(username, coprname))
- mock_chroot = coprs_logic.MockChrootsLogic.get_from_name(chroot, noarch=True).first()
+ mock_chroot = coprs_logic.MockChrootsLogic.get_from_name(name_release, noarch=True).first()
if not mock_chroot:
- return page_not_found("Chroot {0} does not exist".format(chroot))
+ return page_not_found("Chroot {0} does not exist".format(name_release))
url = ""
for build in copr.builds:
diff --git a/frontend/coprs_frontend/coprs/views/stats_ns/stats_receiver.py b/frontend/coprs_frontend/coprs/views/stats_ns/stats_receiver.py
index caf3039..3f811ba 100644
--- a/frontend/coprs_frontend/coprs/views/stats_ns/stats_receiver.py
+++ b/frontend/coprs_frontend/coprs/views/stats_ns/stats_receiver.py
@@ -23,7 +23,7 @@ def increment(counter_type, name):
return "", 201
@stats_rcv_ns.route("/from_logstash", methods=['POST'])
-#@intranet_required
+@intranet_required # ?
def logstash_handler():
# import ipdb; ipdb.set_trace()
json = flask.request.json
9 years, 1 month
[copr] master: [frontend] [rhbz:#1185959] RFE: Present statistics about project popularity : logstash config) (21b5faf)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 21b5fafcd6e31146d47fa3ef56cc43452040680d
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Fri Feb 20 14:39:23 2015 +0100
[frontend] [rhbz:#1185959] RFE: Present statistics about project popularity : logstash config)
>---------------------------------------------------------------
frontend/conf/logstash.conf | 41 +++++++++++++++++++++++++++++++++++++++++
frontend/copr-frontend.spec | 5 +++++
2 files changed, 46 insertions(+), 0 deletions(-)
diff --git a/frontend/conf/logstash.conf b/frontend/conf/logstash.conf
new file mode 100644
index 0000000..82aa01d
--- /dev/null
+++ b/frontend/conf/logstash.conf
@@ -0,0 +1,41 @@
+input {
+ file {
+ path => "/var/log/httpd/access_log"
+ type => "httpd-access"
+ }
+ # file {
+ # path => "/var/log/httpd/error_log"
+ # type => "httpd-error"
+ # }
+}
+
+filter {
+ grok {
+ type => "httpd-access"
+ pattern => "%{COMBINEDAPACHELOG}"
+ }
+}
+
+output {
+ if [type] == "httpd-access" {
+ if ".repo" in [request] and "stats_rcv" not in [request] {
+ http {
+ url => "http://127.0.0.1/stats_rcv/from_logstash"
+ format => "json"
+ http_method => "post"
+ message => "foobar"
+ }
+ }
+ #file {
+ # path => "/var/log/logstash/web.log"
+ # codec => rubydebug {}
+ #
+ #}
+ }
+
+ # file {
+ # path => "/var/log/logstash/all.log"
+ # codec => rubydebug {}
+ # }
+
+}
diff --git a/frontend/copr-frontend.spec b/frontend/copr-frontend.spec
index 49b2b8d..3df1de9 100644
--- a/frontend/copr-frontend.spec
+++ b/frontend/copr-frontend.spec
@@ -57,6 +57,7 @@ Requires: python-flexmock
Requires: python-mock
Requires: python-decorator
Requires: yum
+Requires: logstash
%if 0%{?rhel} < 7 && 0%{?rhel} > 0
BuildRequires: python-argparse
%endif
@@ -128,7 +129,9 @@ touch %{buildroot}%{_sharedstatedir}/copr/data/copr.db
install -d %{buildroot}%{_var}/log/copr
install -d %{buildroot}%{_sysconfdir}/logrotate.d
+install -d %{buildroot}%{_sysconfdir}/logstash.d
cp -a conf/logrotate %{buildroot}%{_sysconfdir}/logrotate.d/%{name}
+cp -a conf/logstash.conf %{buildroot}%{_sysconfdir}/logstash.d/copr_frontend.conf
touch %{buildroot}%{_var}/log/copr/frontend.log
%check
@@ -147,6 +150,7 @@ useradd -r -g copr-fe -G copr-fe -d %{_datadir}/copr/coprs_frontend -s /bin/bash
%post
service httpd condrestart
+service logstash condrestart
%files
%license LICENSE
@@ -157,6 +161,7 @@ service httpd condrestart
%{_datadir}/copr/coprs_frontend
%config(noreplace) %{_sysconfdir}/logrotate.d/%{name}
+%config(noreplace) %{_sysconfdir}/logstash.d/copr_frontend.conf
%defattr(-, copr-fe, copr-fe, -)
%dir %{_sharedstatedir}/copr/data
9 years, 1 month
[copr] master: [frontend] [rhbz:#1185959] RFE: Present statistics about project popularity (WIP todo: package logstash config) (c1a5794)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit c1a5794e39b3d1ea433cccc1c1cb55e52b72fd4a
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Thu Feb 19 23:17:38 2015 +0100
[frontend] [rhbz:#1185959] RFE: Present statistics about project popularity (WIP todo: package logstash config)
>---------------------------------------------------------------
.../450fe5f7942d_added_table_counterstat.py | 31 +++++++
frontend/coprs_frontend/config/copr_devel.conf | 3 +-
frontend/coprs_frontend/coprs/__init__.py | 2 +
frontend/coprs_frontend/coprs/config.py | 1 +
frontend/coprs_frontend/coprs/helpers.py | 7 ++
frontend/coprs_frontend/coprs/logic/stat_logic.py | 88 ++++++++++++++++++++
frontend/coprs_frontend/coprs/models.py | 19 ++++
.../coprs/templates/coprs/detail/overview.html | 6 +-
.../coprs/views/coprs_ns/coprs_general.py | 8 ++-
frontend/coprs_frontend/coprs/views/misc.py | 18 ++++-
.../coprs/views/stats_ns/__init__.py | 4 +
.../coprs/views/stats_ns/stats_receiver.py | 46 ++++++++++
.../tests/test_logic/test_stat_logic.py | 36 ++++++++
13 files changed, 264 insertions(+), 5 deletions(-)
diff --git a/frontend/coprs_frontend/alembic/versions/450fe5f7942d_added_table_counterstat.py b/frontend/coprs_frontend/alembic/versions/450fe5f7942d_added_table_counterstat.py
new file mode 100644
index 0000000..dae6c40
--- /dev/null
+++ b/frontend/coprs_frontend/alembic/versions/450fe5f7942d_added_table_counterstat.py
@@ -0,0 +1,31 @@
+"""added table CounterStat
+
+Revision ID: 450fe5f7942d
+Revises: bd0dab2e478
+Create Date: 2015-02-19 23:40:08.934834
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '450fe5f7942d'
+down_revision = 'bd0dab2e478'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+ ### commands auto generated by Alembic - please adjust! ###
+ op.create_table('counter_stat',
+ sa.Column('name', sa.String(length=127), nullable=False),
+ sa.Column('counter_type', sa.String(length=30), nullable=True),
+ sa.Column('counter', sa.Integer(), server_default='0', nullable=True),
+ sa.PrimaryKeyConstraint('name')
+ )
+ ### end Alembic commands ###
+
+
+def downgrade():
+ ### commands auto generated by Alembic - please adjust! ###
+ op.drop_table('counter_stat')
+ ### end Alembic commands ###
diff --git a/frontend/coprs_frontend/config/copr_devel.conf b/frontend/coprs_frontend/config/copr_devel.conf
index 9f26dc2..d6c5615 100644
--- a/frontend/coprs_frontend/config/copr_devel.conf
+++ b/frontend/coprs_frontend/config/copr_devel.conf
@@ -26,7 +26,7 @@ SQLALCHEMY_DATABASE_URI = 'sqlite:////var/lib/copr/data/copr.db'
#LOGGING_LEVEL = logging.ERROR
DEBUG = True
-SQLALCHEMY_ECHO = True
+SQLALCHEMY_ECHO = False
#CSRF_ENABLED = True
# as of Flask-WTF 0.9+
@@ -44,3 +44,4 @@ ENFORCE_PROTOCOL_FOR_FRONTEND_URL = "http"
PUBLIC_COPR_HOSTNAME = "copr-fe-dev.cloud.fedoraproject.org"
LOG_FILENAME="/tmp/copr_frontend.log"
+INTRANET_IPS = ["127.0.0.1", "192.168.1.102"]
diff --git a/frontend/coprs_frontend/coprs/__init__.py b/frontend/coprs_frontend/coprs/__init__.py
index de5fc5d..492dec3 100644
--- a/frontend/coprs_frontend/coprs/__init__.py
+++ b/frontend/coprs_frontend/coprs/__init__.py
@@ -46,6 +46,7 @@ from coprs.views import status_ns
from coprs.views.status_ns import status_general
from coprs.views import recent_ns
from coprs.views.recent_ns import recent_general
+from coprs.views.stats_ns import stats_receiver
from .context_processors import include_banner
@@ -58,5 +59,6 @@ app.register_blueprint(misc.misc)
app.register_blueprint(backend_ns.backend_ns)
app.register_blueprint(status_ns.status_ns)
app.register_blueprint(recent_ns.recent_ns)
+app.register_blueprint(stats_receiver.stats_rcv_ns)
app.add_url_rule("/", "coprs_ns.coprs_show", coprs_general.coprs_show)
diff --git a/frontend/coprs_frontend/coprs/config.py b/frontend/coprs_frontend/coprs/config.py
index 557dcc6..b8bb37a 100644
--- a/frontend/coprs_frontend/coprs/config.py
+++ b/frontend/coprs_frontend/coprs/config.py
@@ -42,6 +42,7 @@ class Config(object):
# primary log file
LOG_FILENAME = "/var/log/copr/frontend.log"
+ INTRANET_IPS = ["127.0.0.1"]
class ProductionConfig(Config):
diff --git a/frontend/coprs_frontend/coprs/helpers.py b/frontend/coprs_frontend/coprs/helpers.py
index dfa5137..f4f8efa 100644
--- a/frontend/coprs_frontend/coprs/helpers.py
+++ b/frontend/coprs_frontend/coprs/helpers.py
@@ -21,6 +21,13 @@ def generate_api_token(size=30):
return ''.join(random.choice(string.ascii_lowercase) for x in range(size))
+REPO_DL_STAT_FMT = "{user}@{copr}:{name_release}"
+
+
+class CounterStatType(object):
+ REPO_DL = "repo_dl"
+
+
class EnumType(type):
def __call__(self, attr):
diff --git a/frontend/coprs_frontend/coprs/logic/stat_logic.py b/frontend/coprs_frontend/coprs/logic/stat_logic.py
new file mode 100644
index 0000000..c1b2e0f
--- /dev/null
+++ b/frontend/coprs_frontend/coprs/logic/stat_logic.py
@@ -0,0 +1,88 @@
+from collections import defaultdict
+import json
+import os
+import pprint
+import time
+from sqlalchemy import or_
+from sqlalchemy import and_
+
+from sqlalchemy.orm.exc import NoResultFound
+
+from coprs import app
+from coprs import db
+from coprs import exceptions
+from coprs.models import CounterStat
+from coprs import helpers
+from coprs.helpers import REPO_DL_STAT_FMT
+from coprs import signals
+from coprs.helpers import CounterStatType
+
+
+
+
+class CounterStatLogic(object):
+
+ @classmethod
+ def get(cls, name):
+ """
+ :param name: counter name
+ :return:
+ """
+ return CounterStat.query.filter(CounterStat.name == name)
+
+ @classmethod
+ def get_multiply_same_type(cls, counter_type, names_list):
+ return (
+ CounterStat.query
+ .filter(CounterStat.counter_type == counter_type)
+ .filter(CounterStat.name.in_(names_list))
+ )
+
+ @classmethod
+ def add(cls, name, counter_type):
+ csl = CounterStat(name=name, counter_type=counter_type)
+ db.session.add(csl)
+ return csl
+
+ @classmethod
+ def incr(cls, name, counter_type):
+ """
+ Warning: dirty method: does commit if missing stat record.
+ """
+ try:
+ csl = CounterStatLogic.get(name).one()
+ csl.counter = CounterStat.counter + 1
+ except NoResultFound:
+ csl = CounterStatLogic.add(name, counter_type)
+ csl.counter = 1
+
+ db.session.add(csl)
+ return csl
+
+ @classmethod
+ def get_copr_repo_dl_stat(cls, copr):
+ # chroot -> stat_name
+ chroot_by_stat_name = {}
+ for chroot in copr.active_chroots:
+ kwargs = {
+ "user": copr.owner.name,
+ "copr": copr.name,
+ "name_release": chroot.name_release
+ }
+ chroot_by_stat_name[REPO_DL_STAT_FMT.format(**kwargs)] = chroot.name_release
+
+ # [{counter: <value>, name: <stat_name>}, ...]
+ stats = cls.get_multiply_same_type(counter_type=helpers.CounterStatType.REPO_DL,
+ names_list=chroot_by_stat_name.keys())
+
+ # need: {chroot -> value, ... }
+ repo_dl_stats = defaultdict(int)
+ for stat in stats:
+ repo_dl_stats[chroot_by_stat_name[stat.name]] = stat.counter
+
+ return repo_dl_stats
+
+
+
+
+
diff --git a/frontend/coprs_frontend/coprs/models.py b/frontend/coprs_frontend/coprs/models.py
index ef8ad0e..d6cfe5d 100644
--- a/frontend/coprs_frontend/coprs/models.py
+++ b/frontend/coprs_frontend/coprs/models.py
@@ -423,6 +423,13 @@ class MockChroot(db.Model, helpers.Serializer):
return "{0}-{1}-{2}".format(self.os_release, self.os_version, self.arch)
@property
+ def name_release(self):
+ """
+ Textual representation of name of this or release
+ """
+ return "{}-{}".format(self.os_release, self.os_version)
+
+ @property
def os(self):
"""
Textual representation of the operating system name
@@ -578,3 +585,15 @@ class Krb5Login(db.Model, helpers.Serializer):
primary = db.Column(db.String(80), nullable=False, primary_key=True)
user = db.relationship("User", backref=db.backref("krb5_logins"))
+
+
+class CounterStat(db.Model, helpers.Serializer):
+ """
+ Generic store for simple statistics.
+ """
+
+ name = db.Column(db.String(127), primary_key=True)
+ counter_type = db.Column(db.String(30))
+
+ counter = db.Column(db.Integer, default=0, server_default="0")
+
diff --git a/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html b/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html
index 7336814..6063a33 100644
--- a/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html
+++ b/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html
@@ -22,8 +22,10 @@
<tr>
<th class="leftmost">Release</th>
<th>Architecture</th>
- <th class="rightmost">Yum Repo</th>
+
+ <th class="rightmost">Yum Repo <small>[Downloads]</small></th>
</tr>
+ <!-- TODO: remove complex logic from html, group by os-release in the python view -->
{% for mock_chroot in copr.active_chroots %}
{% if loop.index < copr.active_chroots|length %}
{% if mock_chroot.os_release != copr.active_chroots[loop.index].os_release or
@@ -67,8 +69,10 @@
)|fix_url_https_frontend}}">
{{ copr.owner.name }}-{{ copr.name }}-{{mock_chroot.os_release+"-"+mock_chroot.os_version}}.repo
</a>
+ <small>[ {{ repo_dl_stat[mock_chroot.name_release] }} ]</small>
{% endif %}
</td>
+
</tr>
{% else %}
<tr colspan="2"><td>No active releases</td></tr>
diff --git a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py
index be91b69..75ade2f 100644
--- a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py
+++ b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py
@@ -16,6 +16,7 @@ from coprs import exceptions
from coprs import forms
from coprs import helpers
from coprs import models
+from coprs.logic.stat_logic import CounterStatLogic
from coprs.views.misc import login_required, page_not_found
@@ -183,10 +184,13 @@ def copr_detail(username, coprname):
except sqlalchemy.orm.exc.NoResultFound:
return page_not_found(
"Copr with name {0} does not exist.".format(coprname))
-
+ from collections import defaultdict
+ repo_dl_stat = CounterStatLogic.get_copr_repo_dl_stat(copr)
return flask.render_template("coprs/detail/overview.html",
copr=copr,
- form=form)
+ form=form,
+ repo_dl_stat=repo_dl_stat
+ )
@coprs_ns.route("/<username>/<coprname>/permissions/")
diff --git a/frontend/coprs_frontend/coprs/views/misc.py b/frontend/coprs_frontend/coprs/views/misc.py
index 7d6138a..db29c29 100644
--- a/frontend/coprs_frontend/coprs/views/misc.py
+++ b/frontend/coprs_frontend/coprs/views/misc.py
@@ -249,7 +249,23 @@ def backend_authenticated(f):
def decorated_function(*args, **kwargs):
auth = flask.request.authorization
if not auth or auth.password != app.config["BACKEND_PASSWORD"]:
- return "You have to provide the correct password", 401
+ return "You have to provide the correct password\n", 401
+
+ return f(*args, **kwargs)
+ return decorated_function
+
+
+def intranet_required(f):
+ @functools.wraps(f)
+ def decorated_function(*args, **kwargs):
+ # app.logger.debug("MY: {}".format(flask.request.remote_addr))
+ # Use,
+ # >>> import ipaddress
+ # >>> ipaddress.ip_address('192.168.0.1') in ipaddress.ip_network('192.168.0.0/24')
+ if not app.config["DEBUG"]:
+ if flask.request.remote_addr not in app.config["INTRANET_IPS"]:
+ return ("Stats can be update only from intranet hosts, "
+ "not {}, check config\n".format(flask.request.remote_addr)), 403
return f(*args, **kwargs)
return decorated_function
diff --git a/frontend/coprs_frontend/coprs/views/stats_ns/__init__.py b/frontend/coprs_frontend/coprs/views/stats_ns/__init__.py
new file mode 100644
index 0000000..33c09b2
--- /dev/null
+++ b/frontend/coprs_frontend/coprs/views/stats_ns/__init__.py
@@ -0,0 +1,4 @@
+# coding: utf-8
+import flask
+
+stats_rcv_ns = flask.Blueprint("stats_rcv_ns", __name__, url_prefix="/stats_rcv")
diff --git a/frontend/coprs_frontend/coprs/views/stats_ns/stats_receiver.py b/frontend/coprs_frontend/coprs/views/stats_ns/stats_receiver.py
new file mode 100644
index 0000000..caf3039
--- /dev/null
+++ b/frontend/coprs_frontend/coprs/views/stats_ns/stats_receiver.py
@@ -0,0 +1,46 @@
+# coding: utf-8
+
+import flask
+from coprs import app
+from coprs import db
+from coprs.helpers import REPO_DL_STAT_FMT, CounterStatType
+from ..misc import intranet_required
+from . import stats_rcv_ns
+from ...logic.stat_logic import CounterStatLogic
+
+(a)stats_rcv_ns.route("/")
+def ping():
+ return "OK", 200
+
+
+(a)stats_rcv_ns.route("/<counter_type>/<name>/", methods=['POST'])
+@intranet_required
+def increment(counter_type, name):
+ app.logger.debug(flask.request.remote_addr)
+
+ CounterStatLogic.incr(name, counter_type)
+ db.session.commit()
+ return "", 201
+
+(a)stats_rcv_ns.route("/from_logstash", methods=['POST'])
+#@intranet_required
+def logstash_handler():
+ # import ipdb; ipdb.set_trace()
+ json = flask.request.json
+ if "request" in json:
+ # 0 1 2 3 4 5
+ # "request": "/coprs/bob/foox/repo/epel-5/bob-foox-epel-5.repo",
+ req_split = json["request"].split("/")
+ kwargs = dict(
+ user=req_split[2],
+ copr=req_split[3],
+ name_release=req_split[5]
+ )
+ name = REPO_DL_STAT_FMT.format(**kwargs)
+ app.logger.debug("kwargs: {}; name: {}".format(kwargs, name))
+
+ CounterStatLogic.incr(name=name,
+ counter_type=CounterStatType.REPO_DL)
+ db.session.commit()
+
+ return "", 201
diff --git a/frontend/coprs_frontend/tests/test_logic/test_stat_logic.py b/frontend/coprs_frontend/tests/test_logic/test_stat_logic.py
new file mode 100644
index 0000000..14adb49
--- /dev/null
+++ b/frontend/coprs_frontend/tests/test_logic/test_stat_logic.py
@@ -0,0 +1,36 @@
+# coding: utf-8
+import pytest
+
+from coprs.exceptions import ActionInProgressException
+from coprs.helpers import ActionTypeEnum
+from coprs.logic.coprs_logic import CoprsLogic
+from coprs.logic.stat_logic import CounterStatLogic
+from coprs.helpers import CounterStatType
+from coprs import models
+from tests.coprs_test_case import CoprsTestCase
+
+
+class TestStatLogic(CoprsTestCase):
+
+ def setup_method(self, method):
+ super(TestStatLogic, self).setup_method(method)
+
+ self.counter_type = CounterStatType.REPO_DL
+ self.counter_name = "{}:user/copr".format(CounterStatType.REPO_DL)
+
+ def test_counter_basic(self):
+ CounterStatLogic.add(self.counter_name, self.counter_type)
+ self.db.session.commit()
+ CounterStatLogic.incr(self.counter_name, self.counter_type)
+ self.db.session.commit()
+ csl = CounterStatLogic.get(self.counter_name).one()
+ assert csl.counter == 1
+
+ def test_new_by_incr(self):
+ with pytest.raises(Exception):
+ CounterStatLogic.get(self.counter_name).one()
+
+ CounterStatLogic.incr(self.counter_name, self.counter_type)
+ self.db.session.commit()
+ csl = CounterStatLogic.get(self.counter_name).one()
+ assert csl.counter == 1
9 years, 1 month