Uploaded image for project: 'Apache Hudi'
  1. Apache Hudi
  2. HUDI-2506 Hudi dependency governance
  3. HUDI-2440

Add dependency change diff script for dependency governace

    XMLWordPrintableJSON

    Details

      Description

      Currently, hudi's dependency management is chaotic, e.g. for `hudi-spark-bundle_2.11`, the dependency list is here:

      HikariCP/2.5.1//HikariCP-2.5.1.jar
      ST4/4.0.4//ST4-4.0.4.jar
      aircompressor/0.15//aircompressor-0.15.jar
      annotations/17.0.0//annotations-17.0.0.jar
      ant-launcher/1.9.1//ant-launcher-1.9.1.jar
      ant/1.6.5//ant-1.6.5.jar
      ant/1.9.1//ant-1.9.1.jar
      antlr-runtime/3.5.2//antlr-runtime-3.5.2.jar
      aopalliance/1.0//aopalliance-1.0.jar
      apache-curator/2.7.1//apache-curator-2.7.1.pom
      apacheds-i18n/2.0.0-M15//apacheds-i18n-2.0.0-M15.jar
      apacheds-kerberos-codec/2.0.0-M15//apacheds-kerberos-codec-2.0.0-M15.jar
      api-asn1-api/1.0.0-M20//api-asn1-api-1.0.0-M20.jar
      api-util/1.0.0-M20//api-util-1.0.0-M20.jar
      asm/3.1//asm-3.1.jar
      avatica-metrics/1.8.0//avatica-metrics-1.8.0.jar
      avatica/1.8.0//avatica-1.8.0.jar
      avro/1.8.2//avro-1.8.2.jar
      bonecp/0.8.0.RELEASE//bonecp-0.8.0.RELEASE.jar
      calcite-core/1.10.0//calcite-core-1.10.0.jar
      calcite-druid/1.10.0//calcite-druid-1.10.0.jar
      calcite-linq4j/1.10.0//calcite-linq4j-1.10.0.jar
      commons-beanutils-core/1.8.0//commons-beanutils-core-1.8.0.jar
      commons-beanutils/1.7.0//commons-beanutils-1.7.0.jar
      commons-cli/1.2//commons-cli-1.2.jar
      commons-codec/1.4//commons-codec-1.4.jar
      commons-collections/3.2.2//commons-collections-3.2.2.jar
      commons-compiler/2.7.6//commons-compiler-2.7.6.jar
      commons-compress/1.9//commons-compress-1.9.jar
      commons-configuration/1.6//commons-configuration-1.6.jar
      commons-daemon/1.0.13//commons-daemon-1.0.13.jar
      commons-dbcp/1.4//commons-dbcp-1.4.jar
      commons-digester/1.8//commons-digester-1.8.jar
      commons-el/1.0//commons-el-1.0.jar
      commons-httpclient/3.1//commons-httpclient-3.1.jar
      commons-io/2.4//commons-io-2.4.jar
      commons-lang/2.6//commons-lang-2.6.jar
      commons-lang3/3.1//commons-lang3-3.1.jar
      commons-logging/1.2//commons-logging-1.2.jar
      commons-math/2.2//commons-math-2.2.jar
      commons-math3/3.1.1//commons-math3-3.1.1.jar
      commons-net/3.1//commons-net-3.1.jar
      commons-pool/1.5.4//commons-pool-1.5.4.jar
      curator-client/2.7.1//curator-client-2.7.1.jar
      curator-framework/2.7.1//curator-framework-2.7.1.jar
      curator-recipes/2.7.1//curator-recipes-2.7.1.jar
      datanucleus-api-jdo/4.2.4//datanucleus-api-jdo-4.2.4.jar
      datanucleus-core/4.1.17//datanucleus-core-4.1.17.jar
      datanucleus-rdbms/4.1.19//datanucleus-rdbms-4.1.19.jar
      derby/10.10.2.0//derby-10.10.2.0.jar
      disruptor/3.3.0//disruptor-3.3.0.jar
      dropwizard-metrics-hadoop-metrics2-reporter/0.1.2//dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar
      eigenbase-properties/1.1.5//eigenbase-properties-1.1.5.jar
      fastutil/7.0.13//fastutil-7.0.13.jar
      findbugs-annotations/1.3.9-1//findbugs-annotations-1.3.9-1.jar
      fluent-hc/4.4.1//fluent-hc-4.4.1.jar
      groovy-all/2.4.4//groovy-all-2.4.4.jar
      gson/2.3.1//gson-2.3.1.jar
      guava/14.0.1//guava-14.0.1.jar
      guice-assistedinject/3.0//guice-assistedinject-3.0.jar
      guice-servlet/3.0//guice-servlet-3.0.jar
      guice/3.0//guice-3.0.jar
      hadoop-annotations/2.7.3//hadoop-annotations-2.7.3.jar
      hadoop-auth/2.7.3//hadoop-auth-2.7.3.jar
      hadoop-client/2.7.3//hadoop-client-2.7.3.jar
      hadoop-common/2.7.3//hadoop-common-2.7.3.jar
      hadoop-common/2.7.3/tests/hadoop-common-2.7.3-tests.jar
      hadoop-hdfs/2.7.3//hadoop-hdfs-2.7.3.jar
      hadoop-hdfs/2.7.3/tests/hadoop-hdfs-2.7.3-tests.jar
      hadoop-mapreduce-client-app/2.7.3//hadoop-mapreduce-client-app-2.7.3.jar
      hadoop-mapreduce-client-common/2.7.3//hadoop-mapreduce-client-common-2.7.3.jar
      hadoop-mapreduce-client-core/2.7.3//hadoop-mapreduce-client-core-2.7.3.jar
      hadoop-mapreduce-client-jobclient/2.7.3//hadoop-mapreduce-client-jobclient-2.7.3.jar
      hadoop-mapreduce-client-shuffle/2.7.3//hadoop-mapreduce-client-shuffle-2.7.3.jar
      hadoop-yarn-api/2.7.3//hadoop-yarn-api-2.7.3.jar
      hadoop-yarn-client/2.7.3//hadoop-yarn-client-2.7.3.jar
      hadoop-yarn-common/2.7.3//hadoop-yarn-common-2.7.3.jar
      hadoop-yarn-registry/2.7.1//hadoop-yarn-registry-2.7.1.jar
      hadoop-yarn-server-applicationhistoryservice/2.7.2//hadoop-yarn-server-applicationhistoryservice-2.7.2.jar
      hadoop-yarn-server-common/2.7.2//hadoop-yarn-server-common-2.7.2.jar
      hadoop-yarn-server-resourcemanager/2.7.2//hadoop-yarn-server-resourcemanager-2.7.2.jar
      hadoop-yarn-server-web-proxy/2.7.2//hadoop-yarn-server-web-proxy-2.7.2.jar
      hamcrest-core/1.3//hamcrest-core-1.3.jar
      hbase-annotations/1.2.3//hbase-annotations-1.2.3.jar
      hbase-client/1.2.3//hbase-client-1.2.3.jar
      hbase-common/1.2.3//hbase-common-1.2.3.jar
      hbase-common/1.2.3/tests/hbase-common-1.2.3-tests.jar
      hbase-hadoop-compat/1.2.3//hbase-hadoop-compat-1.2.3.jar
      hbase-hadoop2-compat/1.2.3//hbase-hadoop2-compat-1.2.3.jar
      hbase-prefix-tree/1.2.3//hbase-prefix-tree-1.2.3.jar
      hbase-procedure/1.2.3//hbase-procedure-1.2.3.jar
      hbase-protocol/1.2.3//hbase-protocol-1.2.3.jar
      hbase-server/1.2.3//hbase-server-1.2.3.jar
      hive-common/2.3.1//hive-common-2.3.1.jar
      hive-exec/2.3.1//hive-exec-2.3.1.jar
      hive-jdbc/2.3.1//hive-jdbc-2.3.1.jar
      hive-llap-client/2.3.1//hive-llap-client-2.3.1.jar
      hive-llap-common/2.3.1//hive-llap-common-2.3.1.jar
      hive-llap-common/2.3.1/tests/hive-llap-common-2.3.1-tests.jar
      hive-llap-server/2.3.1//hive-llap-server-2.3.1.jar
      hive-llap-tez/2.3.1//hive-llap-tez-2.3.1.jar
      hive-metastore/2.3.1//hive-metastore-2.3.1.jar
      hive-serde/2.3.1//hive-serde-2.3.1.jar
      hive-service-rpc/2.3.1//hive-service-rpc-2.3.1.jar
      hive-service/2.3.1//hive-service-2.3.1.jar
      hive-shims-0.23/2.3.1//hive-shims-0.23-2.3.1.jar
      hive-shims-common/2.3.1//hive-shims-common-2.3.1.jar
      hive-shims-scheduler/2.3.1//hive-shims-scheduler-2.3.1.jar
      hive-shims/2.3.1//hive-shims-2.3.1.jar
      hive-storage-api/2.3.1//hive-storage-api-2.3.1.jar
      hive-vector-code-gen/2.3.1//hive-vector-code-gen-2.3.1.jar
      htrace-core/3.1.0-incubating//htrace-core-3.1.0-incubating.jar
      httpclient/4.4.1//httpclient-4.4.1.jar
      httpcore/4.4.1//httpcore-4.4.1.jar
      ivy/2.4.0//ivy-2.4.0.jar
      jackson-annotations/2.6.7//jackson-annotations-2.6.7.jar
      jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar
      jackson-core/2.6.7//jackson-core-2.6.7.jar
      jackson-databind/2.6.7.3//jackson-databind-2.6.7.3.jar
      jackson-jaxrs/1.9.13//jackson-jaxrs-1.9.13.jar
      jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar
      jackson-module-paranamer/2.7.9//jackson-module-paranamer-2.7.9.jar
      jackson-module-scala_2.11/2.6.7.1//jackson-module-scala_2.11-2.6.7.1.jar
      jackson-xc/1.9.13//jackson-xc-1.9.13.jar
      jamon-runtime/2.3.1//jamon-runtime-2.3.1.jar
      janino/2.7.6//janino-2.7.6.jar
      jasper-compiler/5.5.23//jasper-compiler-5.5.23.jar
      jasper-runtime/5.5.23//jasper-runtime-5.5.23.jar
      java-xmlbuilder/0.4//java-xmlbuilder-0.4.jar
      javalin/2.8.0//javalin-2.8.0.jar
      javax.inject/1//javax.inject-1.jar
      javax.jdo/3.2.0-m3//javax.jdo-3.2.0-m3.jar
      javax.servlet-api/3.1.0//javax.servlet-api-3.1.0.jar
      javax.servlet/3.0.0.v201112011016//javax.servlet-3.0.0.v201112011016.jar
      javolution/5.5.1//javolution-5.5.1.jar
      jaxb-api/2.2.11//jaxb-api-2.2.11.jar
      jaxb-impl/2.2.3-1//jaxb-impl-2.2.3-1.jar
      jcodings/1.0.8//jcodings-1.0.8.jar
      jcommander/1.72//jcommander-1.72.jar
      jdo-api/3.0.1//jdo-api-3.0.1.jar
      jersey-client/1.9//jersey-client-1.9.jar
      jersey-core/1.9//jersey-core-1.9.jar
      jersey-guice/1.9//jersey-guice-1.9.jar
      jersey-json/1.9//jersey-json-1.9.jar
      jersey-server/1.9//jersey-server-1.9.jar
      jets3t/0.9.0//jets3t-0.9.0.jar
      jettison/1.1//jettison-1.1.jar
      jetty-client/9.4.15.v20190215//jetty-client-9.4.15.v20190215.jar
      jetty-http/9.4.15.v20190215//jetty-http-9.4.15.v20190215.jar
      jetty-io/9.4.15.v20190215//jetty-io-9.4.15.v20190215.jar
      jetty-security/9.4.15.v20190215//jetty-security-9.4.15.v20190215.jar
      jetty-server/9.4.15.v20190215//jetty-server-9.4.15.v20190215.jar
      jetty-servlet/9.4.15.v20190215//jetty-servlet-9.4.15.v20190215.jar
      jetty-util/6.1.26//jetty-util-6.1.26.jar
      jetty-util/9.4.15.v20190215//jetty-util-9.4.15.v20190215.jar
      jetty-webapp/9.4.15.v20190215//jetty-webapp-9.4.15.v20190215.jar
      jetty-xml/9.4.15.v20190215//jetty-xml-9.4.15.v20190215.jar
      jetty/6.1.26//jetty-6.1.26.jar
      jline/2.12//jline-2.12.jar
      joda-time/2.9.9//joda-time-2.9.9.jar
      joni/2.1.2//joni-2.1.2.jar
      jpam/1.1//jpam-1.1.jar
      jsch/0.1.42//jsch-0.1.42.jar
      json/1.8//json-1.8.jar
      jsp-api/2.0//jsp-api-2.0.jar
      jsp-api/2.1//jsp-api-2.1.jar
      jsr305/3.0.0//jsr305-3.0.0.jar
      junit/4.12//junit-4.12.jar
      kotlin-stdlib-common/1.2.71//kotlin-stdlib-common-1.2.71.jar
      kotlin-stdlib-jdk7/1.2.71//kotlin-stdlib-jdk7-1.2.71.jar
      kotlin-stdlib-jdk8/1.2.71//kotlin-stdlib-jdk8-1.2.71.jar
      kotlin-stdlib/1.2.71//kotlin-stdlib-1.2.71.jar
      kryo-shaded/4.0.2//kryo-shaded-4.0.2.jar
      leveldbjni-all/1.8//leveldbjni-all-1.8.jar
      libfb303/0.9.3//libfb303-0.9.3.jar
      libthrift/0.9.3//libthrift-0.9.3.jar
      log4j-1.2-api/2.6.2//log4j-1.2-api-2.6.2.jar
      log4j-api/2.6.2//log4j-api-2.6.2.jar
      log4j-core/2.6.2//log4j-core-2.6.2.jar
      log4j-slf4j-impl/2.6.2//log4j-slf4j-impl-2.6.2.jar
      log4j-web/2.6.2//log4j-web-2.6.2.jar
      log4j/1.2.17//log4j-1.2.17.jar
      metrics-core/2.2.0//metrics-core-2.2.0.jar
      metrics-core/4.1.1//metrics-core-4.1.1.jar
      metrics-graphite/4.1.1//metrics-graphite-4.1.1.jar
      metrics-jmx/4.1.1//metrics-jmx-4.1.1.jar
      metrics-json/3.1.0//metrics-json-3.1.0.jar
      metrics-jvm/3.1.0//metrics-jvm-3.1.0.jar
      minlog/1.3.0//minlog-1.3.0.jar
      netty-all/4.0.23.Final//netty-all-4.0.23.Final.jar
      netty/3.6.2.Final//netty-3.6.2.Final.jar
      objenesis/2.5.1//objenesis-2.5.1.jar
      opencsv/2.3//opencsv-2.3.jar
      orc-core/1.3.3//orc-core-1.3.3.jar
      orc-core/1.6.0/nohive/orc-core-1.6.0-nohive.jar
      orc-shims/1.6.0//orc-shims-1.6.0.jar
      oro/2.0.8//oro-2.0.8.jar
      paranamer/2.7//paranamer-2.7.jar
      parquet-avro/1.10.1//parquet-avro-1.10.1.jar
      parquet-column/1.10.1//parquet-column-1.10.1.jar
      parquet-common/1.10.1//parquet-common-1.10.1.jar
      parquet-encoding/1.10.1//parquet-encoding-1.10.1.jar
      parquet-format/2.4.0//parquet-format-2.4.0.jar
      parquet-hadoop-bundle/1.8.1//parquet-hadoop-bundle-1.8.1.jar
      parquet-hadoop/1.10.1//parquet-hadoop-1.10.1.jar
      parquet-jackson/1.10.1//parquet-jackson-1.10.1.jar
      protobuf-java/2.5.0//protobuf-java-2.5.0.jar
      rocksdbjni/5.17.2//rocksdbjni-5.17.2.jar
      scala-library/2.11.12//scala-library-2.11.12.jar
      scala-reflect/2.11.8//scala-reflect-2.11.8.jar
      servlet-api/2.4//servlet-api-2.4.jar
      simpleclient/0.8.0//simpleclient-0.8.0.jar
      simpleclient_common/0.8.0//simpleclient_common-0.8.0.jar
      simpleclient_dropwizard/0.8.0//simpleclient_dropwizard-0.8.0.jar
      simpleclient_httpserver/0.8.0//simpleclient_httpserver-0.8.0.jar
      simpleclient_pushgateway/0.8.0//simpleclient_pushgateway-0.8.0.jar
      slf4j-api/1.7.10//slf4j-api-1.7.10.jar
      slf4j-log4j12/1.7.10//slf4j-log4j12-1.7.10.jar
      slider-core/0.90.2-incubating//slider-core-0.90.2-incubating.jar
      snappy-java/1.1.2.6//snappy-java-1.1.2.6.jar
      spark-avro_2.11/2.4.4//spark-avro_2.11-2.4.4.jar
      spark-tags_2.11/2.4.4//spark-tags_2.11-2.4.4.jar
      stax-api/1.0.1//stax-api-1.0.1.jar
      tephra-api/0.6.0//tephra-api-0.6.0.jar
      tephra-core/0.6.0//tephra-core-0.6.0.jar
      tephra-hbase-compat-1.0/0.6.0//tephra-hbase-compat-1.0-0.6.0.jar
      twill-api/0.6.0-incubating//twill-api-0.6.0-incubating.jar
      twill-common/0.6.0-incubating//twill-common-0.6.0-incubating.jar
      twill-core/0.6.0-incubating//twill-core-0.6.0-incubating.jar
      twill-discovery-api/0.6.0-incubating//twill-discovery-api-0.6.0-incubating.jar
      twill-discovery-core/0.6.0-incubating//twill-discovery-core-0.6.0-incubating.jar
      twill-zookeeper/0.6.0-incubating//twill-zookeeper-0.6.0-incubating.jar
      unused/1.0.0//unused-1.0.0.jar
      velocity/1.5//velocity-1.5.jar
      websocket-api/9.4.15.v20190215//websocket-api-9.4.15.v20190215.jar
      websocket-client/9.4.15.v20190215//websocket-client-9.4.15.v20190215.jar
      websocket-common/9.4.15.v20190215//websocket-common-9.4.15.v20190215.jar
      websocket-server/9.4.15.v20190215//websocket-server-9.4.15.v20190215.jar
      websocket-servlet/9.4.15.v20190215//websocket-servlet-9.4.15.v20190215.jar
      xercesImpl/2.9.1//xercesImpl-2.9.1.jar
      xml-apis/1.3.04//xml-apis-1.3.04.jar
      xmlenc/0.52//xmlenc-0.52.jar
      xz/1.5//xz-1.5.jar
      zookeeper/3.4.6//zookeeper-3.4.6.jar
      zookeeper/3.4.6/tests/zookeeper-3.4.6-tests.jar
      

       

      there are many duplicated dependencies that have different versions. there are many users who report the dependency conflicts problem. We need to review it and build a checker for each PR.

       

        Attachments

          Issue Links

            Activity

              People

              • Assignee:
                yanghua vinoyang
                Reporter:
                yanghua vinoyang
              • Votes:
                0 Vote for this issue
                Watchers:
                1 Start watching this issue

                Dates

                • Created:
                  Updated:
                  Resolved: