summaryrefslogtreecommitdiffstats
path: root/packages_poc_activity.py
blob: 4024bd8d9b13a26d7d3f7accfeb57dd17cb391f7 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
#!/usr/bin/env python -tt
#-*- coding: utf-8 -*-

#    This program is free software; you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation; either version 2 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License along
#    with this program; if not, write to the Free Software Foundation, Inc.,
#    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

"""
This program retrieves the last 50 updates, builds and commits performed
on a certain packages and assert if the current poc should be changed or
not.

Dependencies:
* python-fedmsg-meta-fedora-infrastructure
* python-requests
* python-argparse
* python-progressbar
"""

import datetime
import json
import logging
import pickle
import socket

import fedmsg.config
import fedmsg.meta
import requests
from progressbar import Bar, ETA, Percentage, ProgressBar, RotatingMarker


PKGDB_URL = 'http://209.132.184.188/'
DATAGREPPER_URL = 'https://apps.fedoraproject.org/datagrepper/raw/'
CONFIG = fedmsg.config.load_config([], None)
fedmsg.meta.make_processors(**CONFIG)
PKG_BLACKLIST = ['kernel']

# Initial simple logging stuff
logging.basicConfig()
LOG = logging.getLogger('poc_update')


def get_packages():
    ''' Return a list of all packages in Fedora devel. '''
    LOG.info('Retrieve the list of packages')
    packages = {}
    page = 1
    page_max = 2
    widgets = ['Retrieving packages info: ', Percentage(), ' ',
               Bar(marker=RotatingMarker()), ' ', ETA()]
    pbar = ProgressBar(widgets=widgets, maxval=3).start()
    while page <= page_max:
        args = {
            'branches': 'devel',
            'limit': 100,
            'status': 'Approved',
            'acls': True,
            'page': page,
        }

        timeout = socket.getdefaulttimeout()
        socket.setdefaulttimeout(600)
        try:
            LOG.info("Querying pkgdb")
            output = requests.get(
                '{0}/api/packages'.format(PKGDB_URL), params=args)
        finally:
            socket.setdefaulttimeout(timeout)

        output = json.loads(output.text)

        for pkg in output['packages']:
            for acl in pkg['acls']:
                if acl['collection']['branchname'] == 'devel':
                    packages[pkg['name']] = acl['point_of_contact']
                    break

        page = output['page'] + 1
        page_max = output['pages_total']
        pbar.maxval = page_max + 1
        pbar.update(page)

    pbar.finish()
    LOG.info('{0} packages retrieved'.format(len(packages)))
    return packages


def get_last_actions_on_packages(pkgname):
    """ Query datagrepper to retrieve the last actions performed on a
    package.
    """

    data = {'start': 0,
            'package': [pkgname],
            'topic': [
                'org.fedoraproject.prod.buildsys.build.state.change',
                'org.fedoraproject.prod.bodhi.update.request.testing',
                'org.fedoraproject.prod.git.receive',
            ],
            'rows_per_page': 100,
            'order': 'desc',
            }

    output = requests.get(DATAGREPPER_URL, params=data)

    try:
        json_output = json.loads(output.text)
    except Exception, err:
        print 'ERROR for package %s' % pkgname
        print 'ERROR:', err.message
        print output.url
        return []

    if not json_output['raw_messages']:
        #print 'ERROR no messages for user %s' % username
        return []

    return json_output


def determine_most_active_user(msgs):
    ''' For a given list of fedmsg, extract the user performing the action
    and return the most active user(s).
    '''
    users = {}
    for msg in msgs['raw_messages']:
        if 'Mass_Rebuild' in fedmsg.meta.msg2subtitle(msg):
            continue

        for user in fedmsg.meta.msg2usernames(msg):
            if user == 'ausil' \
                    and msg['topic'].endswith('buildsys.build.state.change'):
                continue
            elif user in users:
                users[user] += 1
            else:
                users[user] = 1

    ranked = {}
    for user in users:
        if users[user] in ranked:
            ranked[users[user]].append(user)
        else:
            ranked[users[user]] = [user]

    keys = sorted(ranked.keys(), reverse=True)
    #print keys[0], ranked[keys[0]]

    if keys:
        return ranked[keys[0]]
    else:
        return []


def main():
    ''' Main section of the program
    Retrieve the list of all packages and for each retrieve the last actions
    they have had and assess if the current poc is correct or not.
    '''

    packages = get_packages()

    output = {}
    no_msg = []

    no_active_poc = []
    no_change = 0
    changes = []
    widgets = ['Processing packages: ', Percentage(), ' ',
               Bar(marker=RotatingMarker()), ' ', ETA()]
    pbar = ProgressBar(widgets=widgets, maxval=len(packages)).start()
    cnt = 0
    output = {}
    for package in packages:
        poc = packages[package]
        cnt = cnt + 1
        pbar.update(cnt)
        if package in PKG_BLACKLIST:
            continue
        msgs = get_last_actions_on_packages(package)
        if not msgs:
            no_msg.append(package)
            continue
        active_poc = determine_most_active_user(msgs)

        if not active_poc:
            no_active_poc.append(package)
        elif poc in active_poc:
            no_change += 1
        else:
            output[package] = {'poc': poc, 'active_poc': active_poc}
            changes.append(
                '%s changes poc from %s to: %s' % (
                    package, poc, ' or '.join(active_poc)
                )
            )
    pbar.finish()
    stream = open('poc_activity.data.pickle', 'w')
    pickle.dump(output, stream)
    stream.close()

    print '%s packages retrieved' % len(packages)
    print '%s packages checked' % cnt
    print '%s packages w/ no package information' % len(no_msg)
    print '%s packages w/ ausil as active point of contact' % len(no_active_poc)
    print "%s packages won't change their point of contact" % no_change
    print "%s packages will change of point of contact" % len(changes)
    print "\nDetails:\n========\n"
    print "\n".join(changes)

if __name__ == '__main__':
    main()