build.gradle.kts

plugins {
    `java-library`
}

ant.importBuild("build.xml") { oldTargetName -> "ant-${oldTargetName}" }

fun antFile(property: String): File = file(ant.properties[property] ?: throw GradleException("Missing $property"))

val buildDir = antFile("build.dir")
val buildClasses = antFile("build.classes")
val buildPlugins by extra { antFile("build.plugins") }

val pluginsDir = antFile("plugins.dir")

val releaseDir = antFile("release.dir")

val testBuildDir = antFile("test.build.dir")
val testBuildClasses = antFile("test.build.classes")
val testBuildLibDir = antFile("test.build.lib.dir")
val testBuildData = antFile("test.build.data")

val confDir by extra { antFile("conf.dir") }

val testRuntimeWorkingDirs = files(confDir, buildPlugins.parent)

java {
    toolchain {
        languageVersion.set(JavaLanguageVersion.of(11))
    }
}

sourceSets {
    main {
        java {
            setSrcDirs(listOf("src/java"))
            destinationDirectory.set(buildClasses)
        }
    }
    test {
        java {
            setSrcDirs(listOf("src/test"))
            destinationDirectory.set(testBuildClasses)
        }
        resources {
            setSrcDirs(listOf("src/test"))
            include("crawl-tests.xml")
            include("domain-urlfilter.txt")
            include("filter-all.txt")
            include("log4j.properties")
            include("nutch-site.xml")
            output.setResourcesDir(testBuildClasses)
        }
    }
}

repositories {
    mavenCentral()
}

dependencies {
    api(libs.hadoop.common) {
        exclude("hsqldb", "hsqldb")
        exclude("net.sf.kosmosfs", "kfs")
        exclude("net.java.dev.jets3t", "jets3t")
        exclude("org.eclipse.jdt", "core")
        exclude("org.mortbay.jetty", "jsp-*")
        exclude("ant", "ant")
        exclude("org.slf4j", "slf4j-log4j12")
    }

    api(libs.apache.commons.jexl3)

    api(libs.tika.core)
    api(libs.xml.apis) // Required by Tika

    api(libs.xerces.impl)

    api(libs.crawler.commons)

    implementation(libs.log4j.api)
    implementation(libs.log4j.core)
    implementation(libs.log4j.slf4j.impl)
    implementation(libs.slf4j.api)

    implementation(libs.apache.commons.lang3)
    implementation(libs.apache.commons.collections4)
    implementation(libs.apache.commons.compress)

    implementation(libs.httpcomponents.httpclient)
    implementation(libs.commons.codec)
    implementation(libs.tdigest)

    implementation(libs.hadoop.hdfs)
    implementation(libs.hadoop.mapreduce.client.core)
    implementation(libs.hadoop.mapreduce.client.jobclient)

    implementation(libs.icu4j)

    implementation(libs.guava)

    implementation(libs.gson)
    implementation(libs.warc.hadoop) {
        exclude(module = "hadoop-client")
    }

    implementation(libs.cxf.rt.frontend.jaxws)
    implementation(libs.cxf.rt.frontend.jaxrs)
    implementation(libs.cxf.rt.transports.http)
    implementation(libs.cxf.rt.transports.http.jetty)
    implementation(libs.jackson.databind)
    implementation(libs.jackson.annotations)
    implementation(libs.jackson.dataformat.cbor)
    implementation(libs.jackson.jaxrs.json.provider)

    // WARC artifacts needed
    implementation(libs.webarchive.commons) {
        exclude(module = "hadoop-core")
        exclude("com.google.guava")
        exclude("junit")
        // Exclude dependencies with incompatible license (see https://www.apache.org/legal/resolved.html#category-x) -->
        exclude("org.json") // JSON License
        // Exclusion of the following dependencies disables support of WARC generation by  "bin/nutch commoncrawldump -warc ..."
        // Please remove these exclusion and recompile Nutch to generate WARC files using the tool "commoncrawldump".
        exclude("it.unimi.dsi", "dsiutils") // LGPL 2.1
        exclude("org.gnu.inet", "libidn") // LGPL 2.1
    }

    // web app dependencies
    implementation(libs.jetty)

    // RabbitMQ dependencies
    implementation(libs.rabbitmq.amqp.client)

    // Added Because of Elasticsearch JEST client
    // TODO refactor these to indexer-elastic-rest plugin somehow, currently doesn't resolve correctly
    implementation(libs.httpcomponents.httpcore.nio)
    implementation(libs.httpcomponents.httpcore)

    implementation(libs.asciitable)

    // artifacts needed for testing
    testImplementation(libs.cxf.rt.rs.client)
    testImplementation(libs.junit)
    testImplementation(libs.mrunit) {
        artifact {
            name = "mrunit"
            classifier = "hadoop2"
        }
        exclude("log4j", "log4j")
    }
    testImplementation(libs.jetty.client)
}

configurations.all {
    // global exclusion
    exclude(module = "jmxtools")
    exclude(module = "jms")
    exclude(module = "jmxri")
    exclude(module = "slf4j-log4j12")
    exclude(module = "log4j") // exclude log4j 1.x
    exclude(group = "com.thoughtworks.xstream")
}

val processOtherResources by tasks.register("processOtherResources") {
    doLast {
        mkdir(buildDir)
        mkdir(buildClasses)
        mkdir(buildPlugins)

        mkdir(releaseDir)

        confDir
            .listFiles { _, name -> name.endsWith(".template") }
            ?.forEach {
                it.setLastModified(33656400) // 01/25/1971 2:00 pm
            }

        copy {
            from(confDir) {
                include("**/*.template")
                rename("""(.*)\.template""", "$1")
            }
            into(confDir)
        }
    }
}

fun Task.dependsOnPluginTask(task: String) {
    subprojects
        .filter { isNutchPlugin(it.path) }
        .forEach { dependsOn(nutchPlugin(it.name).task(task)) }
}

configurations {
    register("testClasses") {
        extendsFrom(testImplementation.get())
    }
}

tasks {
    processResources {
        dependsOn(processOtherResources)
    }

    processTestResources {
        outputs.dirs(testBuildDir, testBuildClasses, testBuildLibDir, testBuildData)
        doLast {
            mkdir(testBuildDir)
            mkdir(testBuildClasses)
            mkdir(testBuildLibDir)
            mkdir(testBuildData)

            copy {
                from("src/testresources")
                into(testBuildData)
            }
        }
    }

    assemble {
        dependsOnPluginTask("assemble")
    }

    test {
        useJUnit()
        dependsOn("assemble")
        dependsOnPluginTask("test")
        systemProperty("test.build.data", testBuildData)
        classpath += testRuntimeWorkingDirs
    }

    clean {
        delete(files("logs"))

        val testFiles = projectDir.listFiles { _, name ->
            name.startsWith("test-crawldb-") || name.startsWith("generate-temp-")
        }
        delete(testFiles)

        val confFiles = confDir.listFiles { _, name -> name.endsWith(".template") }
            ?.map { it.absolutePath.dropLast(9) }
            ?: emptyList()
        delete(confFiles)
    }
}

val testArchive by tasks.register<Jar>("testJar") {
    archiveBaseName.set("apache-nutch-test")
    from(sourceSets["test"].output)
}

artifacts {
    add("testClasses", testArchive)
}