Index: dev-tools/scripts/smokeTestRelease.py =================================================================== --- dev-tools/scripts/smokeTestRelease.py (revision 1390359) +++ dev-tools/scripts/smokeTestRelease.py (working copy) @@ -14,6 +14,7 @@ # limitations under the License. import os +import codecs import tarfile import zipfile import threading @@ -99,8 +100,8 @@ reHREF = re.compile('(.*?)') -# Set to True to avoid re-downloading the packages... -DEBUG = False +# Set to False to avoid re-downloading the packages... +FORCE_CLEAN = True def getHREFs(urlString): @@ -131,7 +132,7 @@ def download(name, urlString, tmpDir, quiet=False): fileName = '%s/%s' % (tmpDir, name) - if DEBUG and os.path.exists(fileName): + if not FORCE_CLEAN and os.path.exists(fileName): if not quiet and fileName.find('.asc') == -1: print(' already done: %.1f MB' % (os.path.getsize(fileName)/1024./1024.)) return @@ -165,15 +166,82 @@ raise RuntimeError('%s contains sheisty class "%s"' % \ (desc, name2)) -def checkAllLuceneJARs(root): +def decodeUTF8(bytes): + return codecs.getdecoder('UTF-8')(bytes)[0] + +MANIFEST_FILE_NAME = 'META-INF/MANIFEST.MF' +NOTICE_FILE_NAME = 'META-INF/NOTICE.txt' +LICENSE_FILE_NAME = 'META-INF/LICENSE.txt' + +def checkJARMetaData(desc, jarFile, version): + + with zipfile.ZipFile(jarFile, 'r') as z: + for name in (MANIFEST_FILE_NAME, NOTICE_FILE_NAME, LICENSE_FILE_NAME): + try: + # The Python docs state a KeyError is raised ... so this None + # check is just defensive: + if z.getinfo(name) is None: + raise RuntimeError('%s is missing %s' % (desc, name)) + except KeyError: + raise RuntimeError('%s is missing %s' % (desc, name)) + + s = decodeUTF8(z.read(MANIFEST_FILE_NAME)) + + for verify in ( + 'Implementation-Vendor: The Apache Software Foundation', + # Make sure 1.6 compiler was used to build release bits: + 'X-Compile-Source-JDK: 1.6', + # Make sure .class files are 1.6 format: + 'X-Compile-Target-JDK: 1.6', + # Make sure this matches the version we think we are releasing: + 'Specification-Version: %s' % version): + if s.find(verify) == -1: + raise RuntimeError('%s is missing "%s" inside its META-INF/MANIFES.MF' % \ + (desc, verify)) + + notice = decodeUTF8(z.read(NOTICE_FILE_NAME)) + license = decodeUTF8(z.read(LICENSE_FILE_NAME)) + + idx = desc.find('inside WAR file') + if idx != -1: + desc2 = desc[:idx] + else: + desc2 = desc + + if desc2.lower().find('solr') != -1: + if SOLR_LICENSE is None: + raise RuntimeError('BUG in smokeTestRelease!') + if SOLR_NOTICE is None: + raise RuntimeError('BUG in smokeTestRelease!') + if notice != SOLR_NOTICE: + raise RuntimeError('%s: %s contents doesn\'t match main NOTICE.txt' % \ + (desc, NOTICE_FILE_NAME)) + if license != SOLR_LICENSE: + raise RuntimeError('%s: %s contents doesn\'t match main LICENSE.txt' % \ + (desc, LICENSE_FILE_NAME)) + else: + if LUCENE_LICENSE is None: + raise RuntimeError('BUG in smokeTestRelease!') + if LUCENE_NOTICE is None: + raise RuntimeError('BUG in smokeTestRelease!') + if notice != LUCENE_NOTICE: + raise RuntimeError('%s: %s contents doesn\'t match main NOTICE.txt' % \ + (desc, NOTICE_FILE_NAME)) + if license != LUCENE_LICENSE: + raise RuntimeError('%s: %s contents doesn\'t match main LICENSE.txt' % \ + (desc, LICENSE_FILE_NAME)) + +def checkAllLuceneJARs(root, version): print(' make sure Lucene JARs don\'t have javax.* or java.* classes...') for root, dirs, files in os.walk(root): for file in files: if file.lower().endswith('.jar'): fullPath = '%s/%s' % (root, file) noJavaPackageClasses('JAR file "%s"' % fullPath, fullPath) + if file.lower().find('lucene') != -1 or file.lower().find('solr') != -1: + checkJARMetaData('JAR file "%s"' % fullPath, fullPath, version) -def checkSolrWAR(warFileName): +def checkSolrWAR(warFileName, version): """ Crawls for JARs inside the WAR and ensures there are no classes @@ -182,11 +250,17 @@ print(' make sure WAR file has no javax.* or java.* classes...') + checkJARMetaData(warFileName, warFileName, version) + with zipfile.ZipFile(warFileName, 'r') as z: for name in z.namelist(): if name.endswith('.jar'): noJavaPackageClasses('JAR file %s inside WAR file %s' % (name, warFileName), io.BytesIO(z.read(name))) + if name.lower().find('lucene') != -1 or name.lower().find('solr') != -1: + checkJARMetaData('JAR file %s inside WAR file %s' % (name, warFileName), + io.BytesIO(z.read(name)), + version) def checkSigs(project, urlString, version, tmpDir, isSigned): @@ -429,7 +503,7 @@ if text == 'Parent Directory' or text == '..': return links[(i+1):] -def unpack(project, tmpDir, artifact, version): +def unpackAndVerify(project, tmpDir, artifact, version): destDir = '%s/unpack' % tmpDir if os.path.exists(destDir): shutil.rmtree(destDir) @@ -454,7 +528,17 @@ unpackPath = '%s/%s' % (destDir, expected) verifyUnpacked(project, artifact, unpackPath, version, tmpDir) +LUCENE_NOTICE = None +LUCENE_LICENSE = None +SOLR_NOTICE = None +SOLR_LICENSE = None + def verifyUnpacked(project, artifact, unpackPath, version, tmpDir): + global LUCENE_NOTICE + global LUCENE_LICENSE + global SOLR_NOTICE + global SOLR_LICENSE + os.chdir(unpackPath) isSrc = artifact.find('-src') != -1 l = os.listdir(unpackPath) @@ -469,6 +553,17 @@ raise RuntimeError('file "%s" is missing from artifact %s' % (fileName, artifact)) l.remove(fileName) + if project == 'lucene': + if LUCENE_NOTICE is None: + LUCENE_NOTICE = open('%s/NOTICE.txt' % unpackPath).read() + if LUCENE_LICENSE is None: + LUCENE_LICENSE = open('%s/LICENSE.txt' % unpackPath).read() + else: + if SOLR_NOTICE is None: + SOLR_NOTICE = open('%s/NOTICE.txt' % unpackPath).read() + if SOLR_LICENSE is None: + SOLR_LICENSE = open('%s/LICENSE.txt' % unpackPath).read() + if not isSrc: # TODO: we should add verifyModule/verifySubmodule (e.g. analysis) here and recurse through if project == 'lucene': @@ -562,11 +657,11 @@ else: if project == 'lucene': - checkAllLuceneJARs(os.getcwd()) + checkAllLuceneJARs(os.getcwd(), version) testDemo(isSrc, version) else: - checkSolrWAR('%s/example/webapps/solr.war' % unpackPath) + checkSolrWAR('%s/example/webapps/solr.war' % unpackPath, version) print(' copying unpacked distribution for Java 6 ...') java6UnpackPath = '%s-java6' %unpackPath @@ -1097,7 +1192,7 @@ os.makedirs(path) crawl(downloadedFiles, subURL, path, exclusions) else: - if not os.path.exists(path) or not DEBUG: + if not os.path.exists(path) or FORCE_CLEAN: download(text, subURL, targetDir, quiet=True) downloadedFiles.append(path) sys.stdout.write('.') @@ -1127,7 +1222,7 @@ def smokeTest(baseURL, version, tmpDir, isSigned): - if not DEBUG: + if FORCE_CLEAN: if os.path.exists(tmpDir): raise RuntimeError('temp dir %s exists; please remove first' % tmpDir) @@ -1158,15 +1253,15 @@ print('Test Lucene...') checkSigs('lucene', lucenePath, version, tmpDir, isSigned) for artifact in ('lucene-%s.tgz' % version, 'lucene-%s.zip' % version): - unpack('lucene', tmpDir, artifact, version) - unpack('lucene', tmpDir, 'lucene-%s-src.tgz' % version, version) + unpackAndVerify('lucene', tmpDir, artifact, version) + unpackAndVerify('lucene', tmpDir, 'lucene-%s-src.tgz' % version, version) print() print('Test Solr...') checkSigs('solr', solrPath, version, tmpDir, isSigned) for artifact in ('apache-solr-%s.tgz' % version, 'apache-solr-%s.zip' % version): - unpack('solr', tmpDir, artifact, version) - unpack('solr', tmpDir, 'apache-solr-%s-src.tgz' % version, version) + unpackAndVerify('solr', tmpDir, artifact, version) + unpackAndVerify('solr', tmpDir, 'apache-solr-%s-src.tgz' % version, version) print('Test Maven artifacts for Lucene and Solr...') checkMaven(baseURL, tmpDir, version, isSigned)