89a90,91
> SITE_CONTENT/podlings_graduated.txt
> SITE_CONTENT/podlings_retired.txt
94a97,98
> # FIXME: Output podling status and project lists as json, yml and ad
> # FIXME: Eliminate the need for the status page and derive it from known content.
101a106,107
> #        2019-03-07 - not sure how viable this is it seems that only one current
> #                     podling - marvin-ai has this problem. hack for now.
141a148
> import json
145c152
< ASF_AUTH_TEMPLATE = 'https://raw.githubusercontent.com/apache/infrastructure-puppet/deployment/modules/subversion_server/files/authorization/asf-authorization-template'
---
> # ASF_AUTH_TEMPLATE = 'https://raw.githubusercontent.com/apache/infrastructure-puppet/deployment/modules/subversion_server/files/authorization/asf-authorization-template'
148a156,164
> # All project git repositories in Apache are organized by Apache Infra and can be found on Gitbox.
> GITBOX_DIR = "https://gitbox.apache.org/repositories.json"
> 
> # All proect committers["members"] and pmc members["owners"] are provided on whimsy
> PROJECT_LDAP = "https://whimsy.apache.org/public/public_ldap_projects.json"
> 
> # All podlings site scan results
> PODLING_SITE = "https://whimsy.apache.org/public/pods-scan.json"
> 
266d281
< newCommitterRE = re.compile("[nN]ew [cC]omm?itt?ers? ?\(?([0-9]+)?")
274a290,303
> # read from directory of Apache Gitbox repositories
> s = urllib.request.urlopen(GITBOX_DIR).read().decode('utf-8')
> gitboxRepos = json.loads(s)
> gitbox = {}
> gitbox["incubator"] = gitboxRepos["projects"]["incubator"]["repositories"]
> 
> # read from project ldap
> s = urllib.request.urlopen(PROJECT_LDAP).read().decode('utf-8')
> projectLDAP = json.loads(s)["projects"]
> 
> # read from podling site scan
> s = urllib.request.urlopen(PODLING_SITE).read().decode('utf-8')
> podlingSiteScan = json.loads(s)
> 
288a318
>     resource = row.getAttribute("resource")
291,292c321
<         resource = row.getAttribute("resource")
<         graduatedProjects[resource.lower()] = {'name': name, 'enddate': endDate}
---
>         graduatedProjects[resource.lower()] = {'name': name, 'startdate': startDate, 'enddate': endDate}
295,296c324
<         resource = row.getAttribute("resource")
<         retiredProjects[resource.lower()] = {'name': name, 'enddate': endDate}
---
>         retiredProjects[resource.lower()] = {'name': name, 'startdate': startDate, 'enddate': endDate}
299c327
<         #print("Name: " + name)
---
>         print("Name: {0} - {1}".format(name,resource.lower()))
302a331,332
>             errorMsg = "00|Duplicate entry in podlings.xml"
>             projects[id]['fixMeList'].append(errorMsg)
309a340,344
>             sid = id
>             if sid == 'marvin-ai':
>                 sid = 'marvin'
>             projects[id]['fixMeList'] = []
>             projects[id]['reposName'] = sid
322a358,365
>             projects[id]['urlGit'] = ""
>             projects[id]['gitbox'] = ""
>             try:
>                 if gitboxRepos["projects"][sid]:
>                     d = gitboxRepos["projects"][sid]["repositories"]
>                     projects[id]['gitbox'] = sorted(d.items(), key=lambda x: x[0])
>             except (Exception) as e:
>                 print("Exception Gitbox repositories " + sid + " : " + str(e))
326a370
>             projects[id]['distributions'] = []
343,344c387,398
<             projects[id]['numberCommitters'] = 0
<             projects[id]['numberCommittersNew'] = 0
---
>             try:
>                 projects[id]['numberCommitters'] = len(projectLDAP[sid]["members"])
>                 projects[id]['numberPMCMembers'] = len(projectLDAP[sid]["owners"])
>             except (Exception) as e:
>                 print("Exception project LDAP " + sid + " : " + str(e))
>                 projects[id]['fixMeList'].append("01|No project LDAP")
>                 projects[id]['numberCommitters'] = 0
>                 projects[id]['numberPMCMembers'] = 0
>             if projects[id]['numberCommitters'] == projects[id]['numberPMCMembers']:
>                 projects[id]['numberPMCEquals'] = True
>             else:
>                 projects[id]['numberPMCEquals'] = False
353a408
>             # FIXME: Get availid from row.getAttribute
363a419,420
>                 errorMsg = "02|Project metadata missing in content/podlings.xml"
>                 projects[id]['fixMeList'].append(errorMsg)
377,378c434
<                     errorMsg = "{0}: Has graduated, but still needs to follow the graduation steps.".format(
<                         id)
---
>                     errorMsg = "{0}: Has graduated, but still needs to follow the graduation steps.".format(id)
381a438,439
>                     errorMsg = "03|Graduated, but some graduation steps are incomplete"
>                     projects[id]['fixMeList'].append(errorMsg)
387,388c445
<                     errorMsg = "{0}: Has retired, but still needs to follow the retirement steps.".format(
<                         id)
---
>                     errorMsg = "{0}: Has retired, but still needs to follow the retirement steps.".format(id)
391a449,450
>                     errorMsg = "04|Retired, but some retirement steps are incomplete"
>                     projects[id]['fixMeList'].append(errorMsg)
398,399c457,461
<                     print(
<                         "ERROR: {0}: expecting a single reportgroup".format(name))
---
>                     errorMsg = "ERROR: {0}: expecting a singleton report group".format(id)
>                     print(errorMsg)
>                     otherIssues.append(errorMsg)
>                     errorMsg = "05|Reporting group is either missing or a multiple"
>                     projects[id]['fixMeList'].append(errorMsg)
407d468
<                     print("ERROR: {0}: missing group attribute".format(name))
408a470,474
>                     errorMsg =" ERROR: {0}: missing group attribute".format(id)
>                     print(errorMsg)
>                     otherIssues.append(errorMsg)
>                     errorMsg = "06|Reporting group is missing"
>                     projects[id]['fixMeList'].append(errorMsg)
491a558,559
>                     if value[-4:] == ".git":
>                         value = ""
493a562,564
>                 if cell.getAttribute('id') == "git" and hasUrl:
>                     projects[k]['urlGit'] = value
>                     continue
500,512d570
<         # Scan the project News section and count new commiters.
<         for section in dom.getElementsByTagName("section"):
<             if 'id' in section.attributes and section.getAttribute('id') == "News":
<                 for line in section.toxml().splitlines():
<                     if ('<!--' in line):
<                         continue
<                     matchNewCommitter = re.search(newCommitterRE, line)
<                     if matchNewCommitter:
<                         if matchNewCommitter.group(1):
<                             projects[k][
<                                 'numberCommittersNew'] += int(matchNewCommitter.group(1))
<                         else:
<                             projects[k]['numberCommittersNew'] += 1
518,543d575
< # Gather committers data ---
< 
< print("Gather committers data ...")
< # Parse the locally defined groups directly
< committers_projects = {}
< with getUrl(ASF_AUTH_TEMPLATE, encoding='UTF-8') as f:
<     for line in f:  # skip the header
<         if line.startswith('[groups]'):
<             break
< 
<     for line in f:  # read the defs section
<         line = line.rstrip()
<         if re.match(r"^(#|\s*$)", line):  # comment or blanks
<             continue
<         if re.match(r"^\[/\]", line):  # end of definition section
<             break
<         m = re.match(r"^\s*(\w\S+?)\s*=\s*(\S+)?$", line)
<         if m:
<             entry = m.group(1)
<             value = m.group(2)
<             if value:  # ignore empty groups
<                 if value.startswith('{'):
<                     continue
<                 committers_projects[entry] = value.split(',')
< # pprint.pprint(committers_projects)
< 
585c617
<     "announce|commits|cvs|dev|issues|notifications|user|users|spec")
---
>     "announce|commits|cvs|dev|issues|notifications|reviews|user|users|spec")
587c619
<     if listName in ["announce", "cvs", "general", "projects"]:
---
>     if listName in ["announce", "cvs", "general", "projects", "dev", "commits", "user"]:
608c640,642
<             print("WARN: Unusual mail list name '{0}'".format(listName))
---
>             errorMsg = "WARN: Unusual mail list name '{0}'".format(listName)
>             print(errorMsg)
>             otherIssues.append(errorMsg)
616,617c650,655
<         print("WARN: {0}: unusual mail list name '{1}', assuming it is their dev list".format(
<             listName, projectMailLists[listName]['dev']))
---
>         errorMsg = "WARN: {0}: unusual mail list name '{1}', assuming it is their dev list".format(
>             listName, projectMailLists[listName]['dev'])
>         print(errorMsg)
>         otherIssues.append(errorMsg)
> #        projects[projectMailLists[listName]]['fixMeList'].append(errorMsg)
> 
628a667
> distributions = {}
640,642c679,687
<                 keysList[
<                     podling] = "{0}/{1}".format("http://www.apache.org/dist/incubator", line)
<             if re.search('(bz2|gz|zip)\.(asc|sig)$', file, flags=re.IGNORECASE):
---
>                 keysList[podling] = "{0}/{1}".format("https://www.apache.org/dist/incubator", line)
>             if re.search('(rpm|msi|bz2|gz|zip)\.(asc|sig)$', file, flags=re.IGNORECASE):
>                 print("{0} - {1}".format(podling,line))
>                 try:
>                     if distributions[podling]:
>                         distributions[podling].append(line)
>                 except:
>                     distributions[podling] = []
>                     distributions[podling].append(line)
646a692,696
>                     if podling in projects:
>                         errorMsg = "INFO: {0}: Release is missing incubator/incubating in file name {1}".format(podling,file)
>                         print(errorMsg)
>                         errorMsg = "42|Distribution is missing incubator/incubating in file name {0}".format(file)
>                         projects[podling]['fixMeList'].append(errorMsg)                    
663,664c713
<     errorMsg = '{0}: Has a distribution filename missing the word "incubating/incubator"'.format(
<         k)
---
>     errorMsg = '{0}: Has a distribution filename missing the word "incubating/incubator"'.format(k)
669,670c718
<         errorMsg = "{0}: Has graduated, but still has remains on Incubator distribution mirrors".format(
<             k)
---
>         errorMsg = "{0}: Has graduated, but still has remains on Incubator distribution mirrors".format(k)
691a740,741
>         errorMsg = "07|Status file is missing"
>         projects[k]['fixMeList'].append(errorMsg)
705c755,758
<             print("ERROR: {0}: ValueError with date".format(k))
---
>             errorMsg = "ERROR: {0}: ValueError with date".format(k)
>             print(errorMsg)
>             errorMsg = "08|Start date is not a valid date"
>             projects[k]['fixMeList'].append(errorMsg)
787,805c840,858
<     svnGroups = projects[k]['resourceNames'][:]
<     sponsor = projects[k]['sponsor'].lower()
<     if '?' in sponsor:
<         sponsor = "incubator"
<     if not 'incubator' in sponsor:
<         tlpSvn = "{0}-{1}".format(sponsor, k)
<         svnGroups.append(tlpSvn)
<     for svnGroup in svnGroups:
<         if optionVerbose:
<             print("DEBUG: Trying committers group '{0}'".format(svnGroup))
<         if svnGroup in committers_projects:
<             projects[k]['numberCommitters'] = len(
<                 committers_projects[svnGroup])
<             projects[k]['committersSvn'] = svnGroup
<             break
<         else:
<             continue
<     if projects[k]['committersSvn'] == None and optionInfo:
<         print("INFO: {0}: Does not yet have committers accounts".format(k))
---
> #    svnGroups = projects[k]['resourceNames'][:]
> #    sponsor = projects[k]['sponsor'].lower()
> #    if '?' in sponsor:
> #        sponsor = "incubator"
> #    if not 'incubator' in sponsor:
> #        tlpSvn = "{0}-{1}".format(sponsor, k)
> #        svnGroups.append(tlpSvn)
> #    for svnGroup in svnGroups:
> #        if optionVerbose:
> #            print("DEBUG: Trying committers group '{0}'".format(svnGroup))
> #        if svnGroup in committers_projects:
> #            projects[k]['numberCommitters'] = len(
> #                committers_projects[svnGroup])
> #            projects[k]['committersSvn'] = svnGroup
> #            break
> #        else:
> #            continue
> #    if projects[k]['committersSvn'] == None and optionInfo:
> #        print("INFO: {0}: Does not yet have committers accounts".format(k))
830c883,886
<             print("INFO: {0}: Does not yet have an Issue Tracker".format(k))
---
>             errorMsg = "INFO: {0}: Does not yet have an Issue Tracker".format(k)
>             print(errorMsg)
>             errorMsg = "10|Project is missing its link to an Issue Tracker"
>             projects[k]['fixMeList'].append(errorMsg)
857c913,977
<             print("INFO: {0}: Does not yet have a website".format(k))
---
>             errorMsg = "INFO: {0}: Does not yet have a website".format(k)
>             print(errorMsg)
>             errorMsg = "11|Website is missing"
>             projects[k]['fixMeList'].append(errorMsg)
>         
>     if projects[k]['urlWww']:
>         # if the website exists then look into site scan for possible issues.
>         j = projects[k]['reposName']
>         try:
>             if podlingSiteScan[j]:
>                 if not podlingSiteScan[j]['foundation']:
>                     errorMsg = "INFO: {0}: Does not yet have foundation link".format(k)
>                     print(errorMsg)
>                     errorMsg = "31|ASF Foundation link is missing"
>                     projects[k]['fixMeList'].append(errorMsg)
>                 if not podlingSiteScan[j]['events']:
>                     errorMsg = "INFO: {0}: Does not yet have events link".format(k)
>                     print(errorMsg)
>                     errorMsg = "32|ASF Events link is missing"
>                     projects[k]['fixMeList'].append(errorMsg)
>                 if not podlingSiteScan[j]['license']:
>                     errorMsg = "INFO: {0}: Does not yet have license link".format(k)
>                     print(errorMsg)
>                     errorMsg = "33|Apache License link is missing"
>                     projects[k]['fixMeList'].append(errorMsg)
>                 if not podlingSiteScan[j]['thanks']:
>                     errorMsg = "INFO: {0}: Does not yet have thanks link".format(k)
>                     print(errorMsg)
>                     errorMsg = "34|Apache Thanks link is missing"
>                     projects[k]['fixMeList'].append(errorMsg)
>                 if not podlingSiteScan[j]['security']:
>                     errorMsg = "INFO: {0}: Does not yet have security link".format(k)
>                     print(errorMsg)
>                     errorMsg = "35|Apache Security link is missing"
>                     projects[k]['fixMeList'].append(errorMsg)
>                 if not podlingSiteScan[j]['sponsorship']:
>                     errorMsg = "INFO: {0}: Does not yet have sponsorship link".format(k)
>                     print(errorMsg)
>                     errorMsg = "36|Apache Sponsorship link is missing"
>                     projects[k]['fixMeList'].append(errorMsg)
>                 if not podlingSiteScan[j]['trademarks']:
>                     errorMsg = "INFO: {0}: Does not yet have trademarks".format(k)
>                     print(errorMsg)
>                     errorMsg = "37|Trademarks are missing"
>                     projects[k]['fixMeList'].append(errorMsg)
>                 if not podlingSiteScan[j]['copyright']:
>                     errorMsg = "INFO: {0}: Does not yet have copyright".format(k)
>                     print(errorMsg)
>                     errorMsg = "38|Copyright is missing"
>                     projects[k]['fixMeList'].append(errorMsg)
>                 flag = False
>                 try:
>                     if not podlingSiteScan[j]['disclaimer']:
>                         flag = True
>                 except (Exception) as e:
>                     flag = True
>                 if flag:
>                     errorMsg = "INFO: {0}: Does not yet have disclaimer".format(k)
>                     print(errorMsg)
>                     errorMsg = "39|Disclaimer is missing"
>                     projects[k]['fixMeList'].append(errorMsg)
>             else:
>                 print('missing from site scan')
>         except (Exception) as e:
>             print("Exception processing : " + str(e))
864,867c984,985
<             urlDist = "http://www.apache.org/dist/incubator/{0}/".format(
<                 nameDist)
<             urlMirror = "http://www.apache.org/dyn/closer.cgi/incubator/{0}/".format(
<                 nameDist)
---
>             urlDist = "https://www.apache.org/dist/incubator/{0}/".format(nameDist)
>             urlMirror = "https://www.apache.org/dyn/closer.cgi/incubator/{0}/".format(nameDist)
873c991,995
<             print("INFO: {0}: Does not yet have a distribution area".format(k))
---
>             errorMsg = "INFO: {0}: Does not yet have a distribution area".format(k)
>             print(errorMsg)
>             errorMsg = "12|Release distribution area is not setup"
>             projects[k]['fixMeList'].append(errorMsg)
> 
883a1006,1012
>                 try:
>                     projects[k]['distributions'] = distributions[nameDistArea]
>                 except KeyError:
>                     errorMsg = "INFO: {0}: Has a PGP KEYS file, but no signed distribution".format(k)
>                     print(errorMsg)
>                     errorMsg = "41|PGP KEYS file available without a distribution"
>                     projects[k]['fixMeList'].append(errorMsg)                    
886,887c1015,1019
<                     print(
<                         "INFO: {0}: Does not yet have a PGP KEYS file".format(k))
---
>                     errorMsg = "INFO: {0}: Does not yet have a PGP KEYS file".format(k)
>                     print(errorMsg)
>                     errorMsg = "13|Release signing PGP KEYS file is missing"
>                     projects[k]['fixMeList'].append(errorMsg)
> 
921c1053
<                 mailListNameDefault = "{0}-{1}".format(projectName, listType)
---
>                 mailListNameDefault = "{0}-{1}-{2}".format(mailListGroup, projectName, listType)
931,932c1063
<                             print("DEBUG: {0}: No incubator group mail lists using '{1}'".format(
<                                 k, projectName))
---
>                             print("DEBUG: {0}: No incubator group mail lists using '{1}'".format(k, projectName))
935,936c1066
<                         leader = 'incubator-' if (
<                             k not in mailLists.newStyle) else ''
---
>                         leader = 'incubator'
938,939c1068
<                             "{0}{1}/".format(leader,
<                                              projectMailLists[projectName][listType])
---
>                             "{0}-{1}/".format(projectName,listType)
947,948c1076,1079
<                             print("INFO: {0}: Does not yet have hinted incubator mail list '{1}-{2}'".format(
<                                 k, projectName, listType))
---
>                             errorMsg = "INFO: {0}: Does not yet have hinted incubator mail list '{1}-{2}'".format(k, projectName, listType)
>                             print(errorMsg)
>                             errorMsg = "14|Expected mailing list (incubator-{0}-{1}) is missing".format(projectName, listType)
>                             projects[k]['fixMeList'].append(errorMsg)
970c1101,1104
<         print("INFO: {0}: Does not yet have 'dev' mail list".format(k))
---
>         errorMsg = "INFO: {0}: Does not yet have 'dev' mail list".format(k)
>         print(errorMsg)
>         errorMsg = "15|Project does not have 'dev' mailing list"
>         projects[k]['fixMeList'].append(errorMsg)
972c1106,1109
<         print("INFO: {0}: Does not yet have 'commits' mail list".format(k))
---
>         errorMsg = "INFO: {0}: Does not yet have 'commits' mail list".format(k)
>         print(errorMsg)
>         errorMsg = "16|Project does not have 'commits' mailing list"
>         projects[k]['fixMeList'].append(errorMsg)
1045a1183
>     persist[k]['reposName'] = projects[k]['reposName']
1136c1274
<         if projects[k]['numberCommitters'] > 2:
---
>         if projects[k]['numberCommitters'] > 5:
1138c1276
<                 projects[k]['committersSvn'], projects[k]['numberCommitters']))
---
>                 projects[k]['reposName'], projects[k]['numberCommitters']))
1141c1279
<                 projects[k]['committersSvn'], projects[k]['numberCommitters']))
---
>                 projects[k]['reposName'], projects[k]['numberCommitters']))
1143a1282
>     persist[k]['numberCommitters'] = projects[k]['numberCommitters']
1145,1148c1284,1287
<     if projects[k]['numberCommittersNew'] > 0:
<         if projects[k]['numberCommittersNew'] > 1:
<             fileXml.write(
<                 '          <td class="cool1 number">{0}</td>\n'.format(projects[k]['numberCommittersNew']))
---
>     if projects[k]['numberPMCMembers'] > 0:
>         if projects[k]['numberPMCMembers'] > 5:
>             fileXml.write('          <td class="cool1 number"><a href="https://people.apache.org/phonebook.html?podling={0}">{1}</a></td>\n'.format(
>                 projects[k]['reposName'], projects[k]['numberPMCMembers']))
1150,1151c1289,1290
<             fileXml.write(
<                 '          <td class="cool2 number">{0}</td>\n'.format(projects[k]['numberCommittersNew']))
---
>             fileXml.write('          <td class="care number"><a href="https://people.apache.org/phonebook.html?podling={0}">{1}</a></td>\n'.format(
>                 projects[k]['reposName'], projects[k]['numberPMCMembers']))
1153c1292,1295
<         fileXml.write('          <td class="care number">0</td>\n')
---
>         fileXml.write('          <td class="care">-</td>\n')
>     persist[k]['numberPMCMembers'] = projects[k]['numberPMCMembers']
> 
>     persist[k]['numberPMCEquals'] = projects[k]['numberPMCEquals']
1159c1301,1308
<         fileXml.write('          <td class="care">False</td>\n')
---
>         if projects[k]['gitbox']:
>             # we have the dictionary of gitbox repositories and we can extract the first git repos from the sorted list.
>             projects[k]['urlGit'] = None
>             projects[k]['urlGit'] = 'https://gitbox.apache.org/repos/asf?p={0}.git;a=tree;hb=HEAD'.format(projects[k]['gitbox'][0][0])
>             fileXml.write(
>                 '          <td class="cool1"><a href="{0}">True</a></td>\n'.format(projects[k]['urlGit']))
>         else:
>             fileXml.write('          <td class="care">False</td>\n')
1160a1310,1312
>     persist[k]['urlGit'] = projects[k]['urlGit']
>     persist[k]['gitbox'] = projects[k]['gitbox']
>     persist[k]['fixMeList'] = projects[k]['fixMeList']
1203a1356,1357
>     persist[k]['urlKeys'] = projects[k]['urlKeys']
>     persist[k]['distributions'] = projects[k]['distributions']
1255a1410,1420
> 
> fileReportG = open(CLUTCH_CONTENT_DIR + 'podlings_graduated.txt', 'w')
> for k in sorted(graduatedProjects, key=str.lower):
>     fileReportG.write("{0},{1},{2}\n".format(k,graduatedProjects[k]['startdate'],graduatedProjects[k]['enddate']))
> fileReportG.close()
> 
> fileReportR = open(CLUTCH_CONTENT_DIR + 'podlings_retired.txt', 'w')
> for k in sorted(retiredProjects, key=str.lower):
>     fileReportR.write("{0},{1},{2}\n".format(k,retiredProjects[k]['startdate'],retiredProjects[k]['enddate']))
> fileReportR.close()
>