From 2c2eb111a020fa9eba20fb9030147f4b39ff2b2c Mon Sep 17 00:00:00 2001 From: Ewen Cheslack-Postava Date: Wed, 3 Jun 2015 22:24:03 -0700 Subject: [PATCH 1/2] KAFKA-2248: Add Rat step to tests to ensure licenses are valid --- .rat-excludes | 26 --- build.gradle | 30 ++- .../test/scala/other/kafka/TestOffsetManager.scala | 17 ++ gradle/buildscript.gradle | 19 +- gradle/license.gradle | 9 - gradle/rat.gradle | 115 ++++++++++++ gradle/resources/rat-output-to-html.xsl | 206 +++++++++++++++++++++ kafka-patch-review.py | 17 ++ scala.gradle | 15 ++ topics.json | 4 - 10 files changed, 412 insertions(+), 46 deletions(-) delete mode 100644 .rat-excludes delete mode 100644 gradle/license.gradle create mode 100644 gradle/rat.gradle create mode 100644 gradle/resources/rat-output-to-html.xsl delete mode 100644 topics.json diff --git a/.rat-excludes b/.rat-excludes deleted file mode 100644 index 01d6298..0000000 --- a/.rat-excludes +++ /dev/null @@ -1,26 +0,0 @@ -.rat-excludes -rat.out -sbt -sbt.boot.lock -README* -.gitignore -.git -.svn -build.properties -target -src_managed -update.log -clients/target -core/target -contrib/target -project/plugins/target -project/build/target -*.iml -*.csproj -TODO -Makefile* -*.html -*.xml -*expected.out -*.kafka - diff --git a/build.gradle b/build.gradle index 30d1cf2..a78a3e7 100644 --- a/build.gradle +++ b/build.gradle @@ -41,8 +41,34 @@ ext { } apply from: file('wrapper.gradle') -apply from: file('gradle/license.gradle') apply from: file('scala.gradle') +apply from: file('gradle/rat.gradle') + +rat { + excludes = [ + '**/.git/**', + '**/.gradle/**', + '**/.project', + '**/.settings', + '**/*.iml', + '**/*.iws', + '**/*.ipr', + '**/.idea/**', + 'gradle/wrapper/**', + '.reviewboardrc', + 'gradlew', + 'gradlew.bat', + '**/README.md', + '**/.DS_Store/**', + '**/.gitignore', + '**/build/**', + 'logs/**', + 'system_test/**', + 'Vagrantfile.local', + '.vagrant/**', + 'core/data/' + ] +} subprojects { apply plugin: 'java' @@ -52,8 +78,6 @@ subprojects { sourceCompatibility = 1.6 - licenseTest.onlyIf { isVerificationRequired(project) } - uploadArchives { repositories { signing { diff --git a/core/src/test/scala/other/kafka/TestOffsetManager.scala b/core/src/test/scala/other/kafka/TestOffsetManager.scala index 8047da4..e0e46c8 100644 --- a/core/src/test/scala/other/kafka/TestOffsetManager.scala +++ b/core/src/test/scala/other/kafka/TestOffsetManager.scala @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package other.kafka import org.I0Itec.zkclient.ZkClient diff --git a/gradle/buildscript.gradle b/gradle/buildscript.gradle index 5e45c06..047632b 100644 --- a/gradle/buildscript.gradle +++ b/gradle/buildscript.gradle @@ -1,3 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + repositories { repositories { // For license plugin. @@ -6,7 +21,3 @@ repositories { } } } - -dependencies { - classpath 'nl.javadude.gradle.plugins:license-gradle-plugin:0.10.0' -} diff --git a/gradle/license.gradle b/gradle/license.gradle deleted file mode 100644 index b4b62eb..0000000 --- a/gradle/license.gradle +++ /dev/null @@ -1,9 +0,0 @@ -subprojects { - apply plugin: 'license' - - license { - header rootProject.file('HEADER') - // Skip Twitter bootstrap JS and CSS. - skipExistingHeaders = true - } -} diff --git a/gradle/rat.gradle b/gradle/rat.gradle new file mode 100644 index 0000000..d62b372 --- /dev/null +++ b/gradle/rat.gradle @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.Task +import org.gradle.api.internal.project.IsolatedAntBuilder + +apply plugin: RatPlugin + +class RatTask extends DefaultTask { + @Input + List excludes + + def reportPath = 'build/rat' + def stylesheet = 'gradle/resources/rat-output-to-html.xsl' + def xmlReport = reportPath + '/rat-report.xml' + def htmlReport = reportPath + '/rat-report.html' + + def generateXmlReport(File reportDir) { + def antBuilder = services.get(IsolatedAntBuilder) + def ratClasspath = project.configurations.rat + antBuilder.withClasspath(ratClasspath).execute { + ant.taskdef(resource: 'org/apache/rat/anttasks/antlib.xml') + ant.report(format: 'xml', reportFile: xmlReport) { + fileset(dir: ".") { + patternset { + excludes.each { + exclude(name: it) + } + } + } + } + } + } + + def printUnknownFiles() { + def ratXml = new XmlParser().parse(xmlReport) + def unknownLicenses = 0 + ratXml.resource.each { resource -> + if (resource.'license-approval'.@name[0] == "false") { + println('Unknown license: ' + resource.@name) + unknownLicenses++ + } + } + if (unknownLicenses > 0) { + throw new GradleException("Found " + unknownLicenses + " files with " + + "unknown licenses.") + } + } + + def generateHtmlReport() { + def antBuilder = services.get(IsolatedAntBuilder) + def ratClasspath = project.configurations.rat + antBuilder.withClasspath(ratClasspath).execute { + ant.xslt( + in: xmlReport, + style: stylesheet, + out: htmlReport, + classpath: ratClasspath) + } + println('Rat report: ' + htmlReport) + } + + @TaskAction + def rat() { + File reportDir = new File(reportPath) + if (!reportDir.exists()) { + reportDir.mkdirs() + } + generateXmlReport(reportDir) + printUnknownFiles() + generateHtmlReport() + } +} + +class RatPlugin implements Plugin { + void apply(Project project) { + configureDependencies(project) + project.plugins.apply(JavaPlugin); + Task ratTask = project.task("rat", + type: RatTask, + group: 'Build', + description: 'Runs Apache Rat checks.') + project.tasks[JavaPlugin.TEST_TASK_NAME].dependsOn ratTask + } + + void configureDependencies(final Project project) { + project.configurations { + rat + } + project.repositories { + mavenCentral() + } + project.dependencies { + rat 'org.apache.rat:apache-rat-tasks:0.11' + } + } +} diff --git a/gradle/resources/rat-output-to-html.xsl b/gradle/resources/rat-output-to-html.xsl new file mode 100644 index 0000000..97ea7a1 --- /dev/null +++ b/gradle/resources/rat-output-to-html.xsl @@ -0,0 +1,206 @@ + + + + + + + + + + + + + + + + + + + + + + +

Rat Report

+

This HTML version (yes, it is!) is generated from the RAT xml reports using Saxon9B. All the outputs required are displayed below, similar to the .txt version. + This is obviously a work in progress; and a prettier, easier to read and manage version will be available soon

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
+Table 1: A snapshot summary of this rat report. +
Notes: Binaries: Archives: Standards:
Apache Licensed: Generated Documents:
Note: JavaDocs are generated and so license header is optionalNote: Generated files do not require license headers
Unknown Licenses - or files without a license. Unknown Licenses - or files without a license.
+
+
+

Unapproved Licenses:

+ + + +
+ + + +
+ +

Archives:

+ + + + +
+ +
+ +

+ Files with Apache License headers will be marked AL
+ Binary files (which do not require AL headers) will be marked B
+ Compressed archives will be marked A
+ Notices, licenses etc will be marked N
+

+ + + + ! + + + + N + A + B + + !!!!! + + +
+ + + +
+ +

Printing headers for files without AL header...

+ + + +

+ +
+ +
+ + + + + + +
+

Resource:

+ +
+ + + + +

First few lines of non-compliant file

+

+ +

+ +

Other Info:

+ + + + Header Type: +
+ + + + License Family: +
+ + + + License Approval: +
+ + + + Type: +
+ + + + + diff --git a/kafka-patch-review.py b/kafka-patch-review.py index b5a2e95..94873c3 100644 --- a/kafka-patch-review.py +++ b/kafka-patch-review.py @@ -1,4 +1,21 @@ #!/usr/bin/env python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. import argparse import sys diff --git a/scala.gradle b/scala.gradle index cabb59c..5eb2a65 100644 --- a/scala.gradle +++ b/scala.gradle @@ -1,3 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + if (!hasProperty('scalaVersion')) { ext.scalaVersion = '2.10.5' } diff --git a/topics.json b/topics.json deleted file mode 100644 index ff011ed..0000000 --- a/topics.json +++ /dev/null @@ -1,4 +0,0 @@ -{"topics": - [{"topic": "foo"}], - "version":1 - } -- 2.4.2 From 0342ce9abf7958640ef914df643e7b17868fcb9a Mon Sep 17 00:00:00 2001 From: Ewen Cheslack-Postava Date: Sat, 13 Jun 2015 14:08:59 -0700 Subject: [PATCH 2/2] KAFKA-2248: Use grgit to make Rat check only consider files that are checked in or staged for check in to git. --- build.gradle | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/build.gradle b/build.gradle index a78a3e7..727d7c5 100644 --- a/build.gradle +++ b/build.gradle @@ -13,11 +13,18 @@ // See the License for the specific language governing permissions and // limitations under the License. +import org.ajoberstar.grgit.Grgit + buildscript { repositories { mavenCentral() } apply from: file('gradle/buildscript.gradle'), to: buildscript + + dependencies { + // For Apache Rat plugin to ignore non-Git files, need ancient version for Java 6 compatibility + classpath group: 'org.ajoberstar', name: 'grgit', version: '0.2.3' + } } def slf4jlog4j='org.slf4j:slf4j-log4j12:1.7.6' @@ -45,29 +52,19 @@ apply from: file('scala.gradle') apply from: file('gradle/rat.gradle') rat { - excludes = [ + // Exclude everything under the directory that git should be ignoring via .gitignore or that isn't checked in. These + // restrict us only to files that are checked in or are staged. + def repo = Grgit.open(project.file('.')) + excludes = new ArrayList(repo.clean(ignore: false, directories: true, dryRun: true)) + // And some of the files that we have checked in should also be excluded from this check + excludes.addAll([ '**/.git/**', - '**/.gradle/**', - '**/.project', - '**/.settings', - '**/*.iml', - '**/*.iws', - '**/*.ipr', - '**/.idea/**', - 'gradle/wrapper/**', - '.reviewboardrc', 'gradlew', 'gradlew.bat', '**/README.md', - '**/.DS_Store/**', - '**/.gitignore', - '**/build/**', - 'logs/**', + '.reviewboardrc', 'system_test/**', - 'Vagrantfile.local', - '.vagrant/**', - 'core/data/' - ] + ]) } subprojects { -- 2.4.2