From bc0ddbd887c5b5eb1e6aec48f98de46964c36645 Mon Sep 17 00:00:00 2001 From: hahwul Date: Sat, 9 Sep 2023 01:22:39 +0900 Subject: [PATCH 1/7] Add missing_option handler in cli --- src/noir.cr | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/noir.cr b/src/noir.cr index 0d42dc55..4518440d 100644 --- a/src/noir.cr +++ b/src/noir.cr @@ -67,6 +67,10 @@ OptionParser.parse do |parser| STDERR.puts parser exit(1) end + parser.missing_option do |flag| + STDERR.puts "ERROR: #{flag} is missing an argument." + exit(1) + end end if noir_options[:base] == "" From ed16bfb1f99f0b71d97929e876e6ea27b8506fbf Mon Sep 17 00:00:00 2001 From: ksg97031 Date: Thu, 7 Sep 2023 22:28:22 +0900 Subject: [PATCH 2/7] Added support for Kotlin Spring and Java Armeria --- .../fixtures/java_armeria/pom.xml | 187 ++++++++ .../java_armeria/src/ApplicationStartUp.java | 84 ++++ .../{spring => java_spring}/build.gradle | 0 .../src/ItemController.java | 0 .../src/MyRoutingConfiguration.java | 0 .../src/QuoteRouter.java | 0 .../fixtures/kotlin_spring/build.gradle.kts | 447 ++++++++++++++++++ .../kotlin_spring/src/UserController.kt | 214 +++++++++ .../testers/java_armeria_spec.cr | 15 + .../testers/java_spring_spec.cr | 2 +- .../testers/kotlin_spring_spec.cr | 19 + .../detector/detect_java_armeria_spec.cr | 13 + src/analyzer/analyzer.cr | 7 +- src/analyzer/analyzers/analyzer_armeria.cr | 64 +++ src/analyzer/analyzers/analyzer_spring.cr | 21 +- src/detector/detector.cr | 2 +- src/detector/detectors/java_armeria.cr | 18 + src/detector/detectors/java_spring.cr | 4 +- src/detector/detectors/kotlin_spring.cr | 15 + src/techs/techs.cr | 10 + 20 files changed, 1108 insertions(+), 14 deletions(-) create mode 100644 spec/functional_test/fixtures/java_armeria/pom.xml create mode 100644 spec/functional_test/fixtures/java_armeria/src/ApplicationStartUp.java rename spec/functional_test/fixtures/{spring => java_spring}/build.gradle (100%) rename spec/functional_test/fixtures/{spring => java_spring}/src/ItemController.java (100%) rename spec/functional_test/fixtures/{spring => java_spring}/src/MyRoutingConfiguration.java (100%) rename spec/functional_test/fixtures/{spring => java_spring}/src/QuoteRouter.java (100%) create mode 100644 spec/functional_test/fixtures/kotlin_spring/build.gradle.kts create mode 100644 spec/functional_test/fixtures/kotlin_spring/src/UserController.kt create mode 100644 spec/functional_test/testers/java_armeria_spec.cr create mode 100644 spec/functional_test/testers/kotlin_spring_spec.cr create mode 100644 spec/unit_test/detector/detect_java_armeria_spec.cr create mode 100644 src/analyzer/analyzers/analyzer_armeria.cr create mode 100644 src/detector/detectors/java_armeria.cr create mode 100644 src/detector/detectors/kotlin_spring.cr diff --git a/spec/functional_test/fixtures/java_armeria/pom.xml b/spec/functional_test/fixtures/java_armeria/pom.xml new file mode 100644 index 00000000..70cc5c58 --- /dev/null +++ b/spec/functional_test/fixtures/java_armeria/pom.xml @@ -0,0 +1,187 @@ + + + + + + apm + org.apache.skywalking + 9.7.0-SNAPSHOT + + 4.0.0 + + apm-webapp + jar + + + UTF-8 + + 1.12.1 + + ${project.parent.basedir}/skywalking-ui + + + + + + org.apache.skywalking + oap-server-bom + ${project.version} + import + pom + + + + + + + org.yaml + snakeyaml + + + + org.slf4j + slf4j-api + + + org.slf4j + log4j-over-slf4j + + + org.apache.logging.log4j + log4j-core + + + org.apache.logging.log4j + log4j-slf4j-impl + + + + io.zipkin + zipkin-lens + + + + com.linecorp.armeria + armeria + + + com.aayushatharva.brotli4j + native-linux-x86_64 + + + com.aayushatharva.brotli4j + native-osx-x86_64 + + + + + + + skywalking-webapp + + + com.github.eirslett + frontend-maven-plugin + ${frontend-maven-plugin.version} + + ${ui.path} + v16.14.0 + + + + install node and npm + + install-node-and-npm + + + + npm ci + + npm + + + ci --registry=https://registry.npmjs.org/ + + + + npm run build + + npm + + + run build + + + + + + maven-resources-plugin + + ${project.build.sourceEncoding} + ${project.build.directory} + + + ${basedir}/target/classes/public + ${ui.path}/dist + + + ${basedir}/target/classes + src/main/resources + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 3.4.2 + + + jar-with-dependencies + + + + org.apache.skywalking.oap.server.webapp.ApplicationStartUp + + + false + + + + make-assembly + package + + single + + + + + + + maven-jar-plugin + + + application.yml + log4j2.xml + + + + + + \ No newline at end of file diff --git a/spec/functional_test/fixtures/java_armeria/src/ApplicationStartUp.java b/spec/functional_test/fixtures/java_armeria/src/ApplicationStartUp.java new file mode 100644 index 00000000..94692715 --- /dev/null +++ b/spec/functional_test/fixtures/java_armeria/src/ApplicationStartUp.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * https://github.com/apache/skywalking/blob/8e529ee95604fb01a8bd31c272763393f3c70525/apm-webapp/src/main/java/org/apache/skywalking/oap/server/webapp/ApplicationStartUp.java#L32 + */ + + package org.apache.skywalking.oap.server.webapp; + + import static org.yaml.snakeyaml.env.EnvScalarConstructor.ENV_FORMAT; + import static org.yaml.snakeyaml.env.EnvScalarConstructor.ENV_TAG; + import java.util.Collections; + import org.yaml.snakeyaml.LoaderOptions; + import org.yaml.snakeyaml.TypeDescription; + import org.yaml.snakeyaml.Yaml; + import org.yaml.snakeyaml.env.EnvScalarConstructor; + import com.linecorp.armeria.common.SessionProtocol; + import com.linecorp.armeria.server.HttpService; + import com.linecorp.armeria.server.Server; + import com.linecorp.armeria.server.file.FileService; + import com.linecorp.armeria.server.file.HttpFile; + import com.linecorp.armeria.server.healthcheck.HealthCheckService; + + public class ApplicationStartUp { + public static void main(String[] args) throws Exception { + final Yaml yaml = new Yaml( + new EnvScalarConstructor( + new TypeDescription(Configuration.class), + Collections.emptyList(), + new LoaderOptions())); + yaml.addImplicitResolver(ENV_TAG, ENV_FORMAT, "$"); + + final Configuration configuration = yaml.loadAs( + ApplicationStartUp.class.getResourceAsStream("/application.yml"), + Configuration.class); + + final int port = configuration.port(); + final String[] oapServices = configuration.oapServices(); + + final HttpService indexPage = + HttpFile + .of(ApplicationStartUp.class.getClassLoader(), "/public/index.html") + .asService(); + final HttpService zipkinIndexPage = + HttpFile + .of(ApplicationStartUp.class.getClassLoader(), "/zipkin-lens/index.html") + .asService(); + + final ZipkinProxyService zipkin = new ZipkinProxyService(configuration.zipkinServices()); + + Server + .builder() + .port(port, SessionProtocol.HTTP) + .service("/graphql", new OapProxyService(oapServices)) + .service("/internal/l7check", HealthCheckService.of()) + .service("/zipkin/config.json", zipkin) + .serviceUnder("/zipkin/api", zipkin) + .serviceUnder("/zipkin", + FileService.of( + ApplicationStartUp.class.getClassLoader(), + "/zipkin-lens") + .orElse(zipkinIndexPage)) + .serviceUnder("/", + FileService.of( + ApplicationStartUp.class.getClassLoader(), + "/public") + .orElse(indexPage)) + .build() + .start() + .join(); + } + } \ No newline at end of file diff --git a/spec/functional_test/fixtures/spring/build.gradle b/spec/functional_test/fixtures/java_spring/build.gradle similarity index 100% rename from spec/functional_test/fixtures/spring/build.gradle rename to spec/functional_test/fixtures/java_spring/build.gradle diff --git a/spec/functional_test/fixtures/spring/src/ItemController.java b/spec/functional_test/fixtures/java_spring/src/ItemController.java similarity index 100% rename from spec/functional_test/fixtures/spring/src/ItemController.java rename to spec/functional_test/fixtures/java_spring/src/ItemController.java diff --git a/spec/functional_test/fixtures/spring/src/MyRoutingConfiguration.java b/spec/functional_test/fixtures/java_spring/src/MyRoutingConfiguration.java similarity index 100% rename from spec/functional_test/fixtures/spring/src/MyRoutingConfiguration.java rename to spec/functional_test/fixtures/java_spring/src/MyRoutingConfiguration.java diff --git a/spec/functional_test/fixtures/spring/src/QuoteRouter.java b/spec/functional_test/fixtures/java_spring/src/QuoteRouter.java similarity index 100% rename from spec/functional_test/fixtures/spring/src/QuoteRouter.java rename to spec/functional_test/fixtures/java_spring/src/QuoteRouter.java diff --git a/spec/functional_test/fixtures/kotlin_spring/build.gradle.kts b/spec/functional_test/fixtures/kotlin_spring/build.gradle.kts new file mode 100644 index 00000000..aa14c5ed --- /dev/null +++ b/spec/functional_test/fixtures/kotlin_spring/build.gradle.kts @@ -0,0 +1,447 @@ +import org.apache.tools.ant.taskdefs.condition.Os +import org.jetbrains.kotlin.gradle.tasks.KotlinCompile +import org.jreleaser.model.Active +import org.jreleaser.model.Distribution.DistributionType.SINGLE_JAR +import org.jreleaser.model.api.common.Apply + +plugins { + run { + kotlin("jvm") + kotlin("plugin.spring") + kotlin("kapt") + } + id("org.springframework.boot") version "3.1.1" + id("com.gorylenko.gradle-git-properties") version "2.4.1" + id("nu.studer.jooq") version "8.2.1" + id("org.flywaydb.flyway") version "9.7.0" + id("com.github.johnrengelman.processes") version "0.5.0" + id("org.springdoc.openapi-gradle-plugin") version "1.6.0" + id("org.jreleaser") version "1.6.0" + id("com.google.devtools.ksp") version "1.8.22-1.0.11" + + jacoco +} + +group = "org.gotson" + +val benchmarkSourceSet = sourceSets.create("benchmark") { + java { + compileClasspath += sourceSets.main.get().output + runtimeClasspath += sourceSets.main.get().runtimeClasspath + } +} + +val benchmarkImplementation by configurations.getting { + extendsFrom(configurations.testImplementation.get()) +} +val kaptBenchmark by configurations.getting { + extendsFrom(configurations.kaptTest.get()) +} + +dependencies { + implementation(kotlin("stdlib")) + implementation(kotlin("reflect")) + + implementation(platform("org.springframework.boot:spring-boot-dependencies:3.1.1")) + + implementation("org.springframework.boot:spring-boot-starter-web") + implementation("org.springframework.boot:spring-boot-starter-webflux") + implementation("org.springframework.boot:spring-boot-starter-validation") + implementation("org.springframework.boot:spring-boot-starter-actuator") + implementation("org.springframework.boot:spring-boot-starter-security") + implementation("org.springframework.boot:spring-boot-starter-oauth2-client") + implementation("org.springframework.boot:spring-boot-starter-thymeleaf") + implementation("org.springframework.boot:spring-boot-starter-artemis") + implementation("org.springframework.boot:spring-boot-starter-jooq") + implementation("org.springframework.session:spring-session-core") + implementation("com.github.gotson:spring-session-caffeine:2.0.0") + implementation("org.springframework.data:spring-data-commons") + + kapt("org.springframework.boot:spring-boot-configuration-processor:3.1.1") + + implementation("org.apache.activemq:artemis-jakarta-server") + + implementation("org.flywaydb:flyway-core") + + implementation("io.github.microutils:kotlin-logging-jvm:3.0.5") + implementation("io.hawt:hawtio-springboot:2.17.4") + + implementation("org.springdoc:springdoc-openapi-starter-webmvc-ui:2.1.0") + + implementation("com.fasterxml.jackson.module:jackson-module-kotlin") + implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-xml") + + implementation("commons-io:commons-io:2.13.0") + implementation("org.apache.commons:commons-lang3:3.12.0") + implementation("commons-validator:commons-validator:1.7") + + run { + val luceneVersion = "9.7.0" + implementation("org.apache.lucene:lucene-core:$luceneVersion") + implementation("org.apache.lucene:lucene-analysis-common:$luceneVersion") + implementation("org.apache.lucene:lucene-queryparser:$luceneVersion") + implementation("org.apache.lucene:lucene-backward-codecs:$luceneVersion") + } + + implementation("com.ibm.icu:icu4j:73.2") + + implementation("com.appmattus.crypto:cryptohash:0.10.1") + + implementation("org.apache.tika:tika-core:2.8.0") + implementation("org.apache.commons:commons-compress:1.23.0") + implementation("com.github.junrar:junrar:7.5.4") + implementation("org.apache.pdfbox:pdfbox:2.0.28") + implementation("net.grey-panther:natural-comparator:1.1") + implementation("org.jsoup:jsoup:1.16.1") + + implementation("net.coobird:thumbnailator:0.4.19") + runtimeOnly("com.twelvemonkeys.imageio:imageio-jpeg:3.9.4") + runtimeOnly("com.twelvemonkeys.imageio:imageio-tiff:3.9.4") + runtimeOnly("com.twelvemonkeys.imageio:imageio-webp:3.9.4") + runtimeOnly("com.github.gotson.nightmonkeys:imageio-jxl:0.4.1") + // support for jpeg2000 + runtimeOnly("com.github.jai-imageio:jai-imageio-jpeg2000:1.4.0") + runtimeOnly("org.apache.pdfbox:jbig2-imageio:3.0.4") + + // barcode scanning + implementation("com.google.zxing:core:3.5.1") + + implementation("com.jakewharton.byteunits:byteunits:0.9.1") + + implementation("com.github.f4b6a3:tsid-creator:5.2.4") + + implementation("com.github.ben-manes.caffeine:caffeine") + + implementation("org.xerial:sqlite-jdbc:3.42.0.0") + jooqGenerator("org.xerial:sqlite-jdbc:3.42.0.0") + + if (version.toString().endsWith(".0.0")) { + ksp("com.github.gotson.bestbefore:bestbefore-processor-kotlin:0.1.0") + } + + testImplementation("org.springframework.boot:spring-boot-starter-test") { + exclude(module = "mockito-core") + } + testImplementation("org.springframework.security:spring-security-test") + testImplementation("com.ninja-squad:springmockk:4.0.2") + testImplementation("io.mockk:mockk:1.13.5") + testImplementation("com.google.jimfs:jimfs:1.2") + + testImplementation("com.tngtech.archunit:archunit-junit5:1.0.1") + + benchmarkImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.1") + benchmarkImplementation("org.openjdk.jmh:jmh-core:1.36") + kaptBenchmark("org.openjdk.jmh:jmh-generator-annprocess:1.36") + kaptBenchmark("org.springframework.boot:spring-boot-configuration-processor:3.1.1") + + developmentOnly("org.springframework.boot:spring-boot-devtools:3.1.1") +} + +val webui = "$rootDir/komga-webui" +tasks { + withType { + sourceCompatibility = "17" + targetCompatibility = "17" + } + withType { + kotlinOptions { + jvmTarget = "17" + freeCompilerArgs = listOf( + "-Xjsr305=strict", + "-opt-in=kotlin.time.ExperimentalTime", + ) + } + } + + withType { + useJUnitPlatform() + systemProperty("spring.profiles.active", "test") + maxHeapSize = "1G" + } + + getByName("jar") { + enabled = false + } + + register("npmInstall") { + group = "web" + workingDir(webui) + inputs.file("$webui/package.json") + outputs.dir("$webui/node_modules") + commandLine( + if (Os.isFamily(Os.FAMILY_WINDOWS)) { + "npm.cmd" + } else { + "npm" + }, + "install", + ) + } + + register("npmBuild") { + group = "web" + dependsOn("npmInstall") + workingDir(webui) + inputs.dir(webui) + outputs.dir("$webui/dist") + commandLine( + if (Os.isFamily(Os.FAMILY_WINDOWS)) { + "npm.cmd" + } else { + "npm" + }, + "run", + "build", + ) + } + + // copy the webui build into public + register("copyWebDist") { + group = "web" + dependsOn("npmBuild") + from("$webui/dist/") + into("$projectDir/src/main/resources/public/") + } + + withType { + filesMatching("application*.yml") { + expand(project.properties) + } + mustRunAfter(getByName("copyWebDist")) + } + + register("benchmark") { + group = "benchmark" + inputs.files(benchmarkSourceSet.output) + testClassesDirs = benchmarkSourceSet.output.classesDirs + classpath = benchmarkSourceSet.runtimeClasspath + } +} + +springBoot { + buildInfo { + // prevent task bootBuildInfo to rerun every time + excludes.set(setOf("time")) + properties { + // but rerun if the gradle.properties file changed + inputs.file("$rootDir/gradle.properties") + } + } +} + +sourceSets { + // add a flyway sourceSet + val flyway by creating { + compileClasspath += sourceSets.main.get().compileClasspath + runtimeClasspath += sourceSets.main.get().runtimeClasspath + } + // main sourceSet depends on the output of flyway sourceSet + main { + output.dir(flyway.output) + } +} + +val dbSqlite = mapOf( + "url" to "jdbc:sqlite:${project.buildDir}/generated/flyway/database.sqlite", +) +val migrationDirsSqlite = listOf( + "$projectDir/src/flyway/resources/db/migration/sqlite", + "$projectDir/src/flyway/kotlin/db/migration/sqlite", +) +flyway { + url = dbSqlite["url"] + locations = arrayOf("classpath:db/migration/sqlite") + placeholders = mapOf("library-file-hashing" to "true") +} +tasks.flywayMigrate { + // in order to include the Java migrations, flywayClasses must be run before flywayMigrate + dependsOn("flywayClasses") + migrationDirsSqlite.forEach { inputs.dir(it) } + outputs.dir("${project.buildDir}/generated/flyway") + doFirst { + delete(outputs.files) + mkdir("${project.buildDir}/generated/flyway") + } + mixed = true +} + +jooq { + version.set("3.17.4") + configurations { + create("main") { + jooqConfiguration.apply { + logging = org.jooq.meta.jaxb.Logging.WARN + jdbc.apply { + driver = "org.sqlite.JDBC" + url = dbSqlite["url"] + } + generator.apply { + database.apply { + name = "org.jooq.meta.sqlite.SQLiteDatabase" + } + target.apply { + packageName = "org.gotson.komga.jooq" + } + } + } + } + } +} +tasks.named("generateJooq") { + migrationDirsSqlite.forEach { inputs.dir(it) } + allInputsDeclared.set(true) + dependsOn("flywayMigrate") +} + +openApi { + outputDir.set(file("$projectDir/docs")) + customBootRun { + args.add("--spring.profiles.active=claim") + args.add("--server.port=8080") + } +} + +tasks.jacocoTestReport { + dependsOn(tasks.test) +} + +configure { + filter { + exclude("**/db/migration/**") + } +} + +jreleaser { + gitRootSearch.set(true) + + project { + description.set("Media server for comics/mangas/BDs with API and OPDS support") + copyright.set("Gauthier Roebroeck") + authors.add("Gauthier Roebroeck") + license.set("MIT") + links { + homepage.set("https://komga.org") + } + } + + release { + github { + discussionCategoryName.set("Announcements") + + changelog { + formatted.set(Active.ALWAYS) + preset.set("conventional-commits") + skipMergeCommits.set(true) + links.set(true) + format.set("- {{#commitIsConventional}}{{#conventionalCommitIsBreakingChange}}🚨 {{/conventionalCommitIsBreakingChange}}{{#conventionalCommitScope}}**{{conventionalCommitScope}}**: {{/conventionalCommitScope}}{{conventionalCommitDescription}}{{#conventionalCommitBreakingChangeContent}}: *{{conventionalCommitBreakingChangeContent}}*{{/conventionalCommitBreakingChangeContent}} ({{commitShortHash}}){{/commitIsConventional}}{{^commitIsConventional}}{{commitTitle}} ({{commitShortHash}}){{/commitIsConventional}}{{#commitHasIssues}}, closes{{#commitIssues}} {{issue}}{{/commitIssues}}{{/commitHasIssues}}") + hide { + uncategorized.set(true) + contributors.set(listOf("Weblate", "GitHub", "semantic-release-bot", "[bot]", "github-actions")) + } + excludeLabels.add("chore") + category { + title.set("🏎 Perf") + key.set("perf") + labels.add("perf") + order.set(25) + } + category { + title.set("🌐 Translation") + key.set("i18n") + labels.add("i18n") + order.set(70) + } + labeler { + label.set("perf") + title.set("regex:^(?:perf(?:\\(.*\\))?!?):\\s.*") + order.set(120) + } + labeler { + label.set("i18n") + title.set("regex:^(?:i18n(?:\\(.*\\))?!?):\\s.*") + order.set(130) + } + extraProperties.put("categorizeScopes", true) + append { + enabled.set(true) + title.set("# [{{projectVersion}}]({{repoUrl}}/compare/{{previousTagName}}...{{tagName}}) ({{#f_now}}YYYY-MM-dd{{/f_now}})") + target.set(rootDir.resolve("CHANGELOG.md")) + content.set( + """ + {{changelogTitle}} + {{changelogChanges}} + """.trimIndent(), + ) + } + } + + issues { + enabled.set(true) + comment.set("🎉 This issue has been resolved in `{{tagName}}` ([Release Notes]({{releaseNotesUrl}}))") + applyMilestone.set(Apply.ALWAYS) + label { + name.set("released") + description.set("Issue has been released") + color.set("#ededed") + } + } + } + } + + checksum.individual.set(true) + + distributions { + create("komga") { + distributionType.set(SINGLE_JAR) + artifact { + path.set(files(tasks.bootJar).singleFile) + } + } + } + + packagers { + docker { + active.set(Active.RELEASE) + continueOnError.set(true) + templateDirectory.set(projectDir.resolve("docker")) + repository.active.set(Active.NEVER) + buildArgs.set(listOf("--cache-from", "gotson/komga:latest")) + imageNames.set( + listOf( + "komga:latest", + "komga:{{projectVersion}}", + "komga:{{projectVersionMajor}}.x", + ), + ) + registries { + create("docker.io") { externalLogin.set(true) } + create("ghcr.io") { externalLogin.set(true) } + } + buildx { + enabled.set(true) + createBuilder.set(false) + platforms.set( + listOf( + "linux/amd64", + "linux/arm/v7", + "linux/arm64/v8", + ), + ) + } + } + } +} + +project.afterEvaluate { + tasks.named("forkedSpringBootRun") { + mustRunAfter(tasks.bootJar) + } +} + +tasks.jreleaserPackage { + inputs.files(tasks.bootJar) +} +// Workaround for https://github.com/jreleaser/jreleaser/issues/1231 +tasks.jreleaserFullRelease { + inputs.files(tasks.bootJar) +} \ No newline at end of file diff --git a/spec/functional_test/fixtures/kotlin_spring/src/UserController.kt b/spec/functional_test/fixtures/kotlin_spring/src/UserController.kt new file mode 100644 index 00000000..4b8d0755 --- /dev/null +++ b/spec/functional_test/fixtures/kotlin_spring/src/UserController.kt @@ -0,0 +1,214 @@ +// https://github.com/gotson/komga/blob/e50591f372f0ac077bbaf730b1439220a32608af/komga/src/main/kotlin/org/gotson/komga/interfaces/api/rest/UserController.kt +package org.gotson.komga.interfaces.api.rest + +import io.swagger.v3.oas.annotations.Parameter +import jakarta.validation.Valid +import mu.KotlinLogging +import org.gotson.komga.domain.model.AgeRestriction +import org.gotson.komga.domain.model.ContentRestrictions +import org.gotson.komga.domain.model.ROLE_ADMIN +import org.gotson.komga.domain.model.ROLE_FILE_DOWNLOAD +import org.gotson.komga.domain.model.ROLE_PAGE_STREAMING +import org.gotson.komga.domain.model.UserEmailAlreadyExistsException +import org.gotson.komga.domain.persistence.AuthenticationActivityRepository +import org.gotson.komga.domain.persistence.KomgaUserRepository +import org.gotson.komga.domain.persistence.LibraryRepository +import org.gotson.komga.domain.service.KomgaUserLifecycle +import org.gotson.komga.infrastructure.jooq.UnpagedSorted +import org.gotson.komga.infrastructure.security.KomgaPrincipal +import org.gotson.komga.interfaces.api.rest.dto.AuthenticationActivityDto +import org.gotson.komga.interfaces.api.rest.dto.PasswordUpdateDto +import org.gotson.komga.interfaces.api.rest.dto.UserCreationDto +import org.gotson.komga.interfaces.api.rest.dto.UserDto +import org.gotson.komga.interfaces.api.rest.dto.UserUpdateDto +import org.gotson.komga.interfaces.api.rest.dto.toDto +import org.springdoc.core.converters.models.PageableAsQueryParam +import org.springframework.core.env.Environment +import org.springframework.data.domain.Page +import org.springframework.data.domain.PageRequest +import org.springframework.data.domain.Pageable +import org.springframework.data.domain.Sort +import org.springframework.http.HttpStatus +import org.springframework.http.MediaType +import org.springframework.security.access.prepost.PreAuthorize +import org.springframework.security.core.annotation.AuthenticationPrincipal +import org.springframework.web.bind.annotation.DeleteMapping +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PatchMapping +import org.springframework.web.bind.annotation.PathVariable +import org.springframework.web.bind.annotation.PostMapping +import org.springframework.web.bind.annotation.RequestBody +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RequestParam +import org.springframework.web.bind.annotation.ResponseStatus +import org.springframework.web.bind.annotation.RestController +import org.springframework.web.server.ResponseStatusException + +private val logger = KotlinLogging.logger {} + +@RestController +@RequestMapping("/api/v2/users", produces = [MediaType.APPLICATION_JSON_VALUE]) +class UserController( + private val userLifecycle: KomgaUserLifecycle, + private val userRepository: KomgaUserRepository, + private val libraryRepository: LibraryRepository, + private val authenticationActivityRepository: AuthenticationActivityRepository, + env: Environment, +) { + + private val demo = env.activeProfiles.contains("demo") + + @GetMapping("/me") + fun getMe(@AuthenticationPrincipal principal: KomgaPrincipal): UserDto = + principal.toDto() + + @PatchMapping("/me/password") + @ResponseStatus(HttpStatus.NO_CONTENT) + fun updateMyPassword( + @AuthenticationPrincipal principal: KomgaPrincipal, + @Valid @RequestBody + newPasswordDto: PasswordUpdateDto, + ) { + if (demo) throw ResponseStatusException(HttpStatus.FORBIDDEN) + userRepository.findByEmailIgnoreCaseOrNull(principal.username)?.let { user -> + userLifecycle.updatePassword(user, newPasswordDto.password, false) + } ?: throw ResponseStatusException(HttpStatus.NOT_FOUND) + } + + @GetMapping + @PreAuthorize("hasRole('$ROLE_ADMIN')") + fun getAll(): List = + userRepository.findAll().map { it.toDto() } + + @PostMapping + @ResponseStatus(HttpStatus.CREATED) + @PreAuthorize("hasRole('$ROLE_ADMIN')") + fun addOne( + @Valid @RequestBody + newUser: UserCreationDto, + ): UserDto = + try { + userLifecycle.createUser(newUser.toDomain()).toDto() + } catch (e: UserEmailAlreadyExistsException) { + throw ResponseStatusException(HttpStatus.BAD_REQUEST, "A user with this email already exists") + } + + @DeleteMapping("/{id}") + @ResponseStatus(HttpStatus.NO_CONTENT) + @PreAuthorize("hasRole('$ROLE_ADMIN') and #principal.user.id != #id") + fun delete( + @PathVariable id: String, + @AuthenticationPrincipal principal: KomgaPrincipal, + ) { + userRepository.findByIdOrNull(id)?.let { + userLifecycle.deleteUser(it) + } ?: throw ResponseStatusException(HttpStatus.NOT_FOUND) + } + + @PatchMapping("/{id}") + @ResponseStatus(HttpStatus.NO_CONTENT) + @PreAuthorize("hasRole('$ROLE_ADMIN') and #principal.user.id != #id") + fun updateUser( + @PathVariable id: String, + @Valid @RequestBody + patch: UserUpdateDto, + @AuthenticationPrincipal principal: KomgaPrincipal, + ) { + userRepository.findByIdOrNull(id)?.let { existing -> + val updatedUser = with(patch) { + existing.copy( + roleAdmin = if (isSet("roles")) roles!!.contains(ROLE_ADMIN) else existing.roleAdmin, + roleFileDownload = if (isSet("roles")) roles!!.contains(ROLE_FILE_DOWNLOAD) else existing.roleFileDownload, + rolePageStreaming = if (isSet("roles")) roles!!.contains(ROLE_PAGE_STREAMING) else existing.rolePageStreaming, + sharedAllLibraries = if (isSet("sharedLibraries")) sharedLibraries!!.all else existing.sharedAllLibraries, + sharedLibrariesIds = if (isSet("sharedLibraries")) { + if (sharedLibraries!!.all) emptySet() + else libraryRepository.findAllByIds(sharedLibraries!!.libraryIds).map { it.id }.toSet() + } else existing.sharedLibrariesIds, + restrictions = ContentRestrictions( + ageRestriction = if (isSet("ageRestriction")) { + if (ageRestriction == null) null + else AgeRestriction(ageRestriction!!.age, ageRestriction!!.restriction) + } else existing.restrictions.ageRestriction, + labelsAllow = if (isSet("labelsAllow")) labelsAllow + ?: emptySet() else existing.restrictions.labelsAllow, + labelsExclude = if (isSet("labelsExclude")) labelsExclude + ?: emptySet() else existing.restrictions.labelsExclude, + ), + ) + } + userLifecycle.updateUser(updatedUser) + } ?: throw ResponseStatusException(HttpStatus.NOT_FOUND) + } + + @PatchMapping("/{id}/password") + @ResponseStatus(HttpStatus.NO_CONTENT) + @PreAuthorize("hasRole('$ROLE_ADMIN') or #principal.user.id == #id") + fun updatePassword( + @PathVariable id: String, + @AuthenticationPrincipal principal: KomgaPrincipal, + @Valid @RequestBody + newPasswordDto: PasswordUpdateDto, + ) { + if (demo) throw ResponseStatusException(HttpStatus.FORBIDDEN) + userRepository.findByIdOrNull(id)?.let { user -> + userLifecycle.updatePassword(user, newPasswordDto.password, user.id != principal.user.id) + } ?: throw ResponseStatusException(HttpStatus.NOT_FOUND) + } + + @GetMapping("/me/authentication-activity") + @PageableAsQueryParam + fun getMyAuthenticationActivity( + @AuthenticationPrincipal principal: KomgaPrincipal, + @RequestParam(name = "unpaged", required = false) unpaged: Boolean = false, + @Parameter(hidden = true) page: Pageable, + ): Page { + if (demo && !principal.user.roleAdmin) throw ResponseStatusException(HttpStatus.FORBIDDEN) + val sort = + if (page.sort.isSorted) page.sort + else Sort.by(Sort.Order.desc("dateTime")) + + val pageRequest = + if (unpaged) UnpagedSorted(sort) + else PageRequest.of( + page.pageNumber, + page.pageSize, + sort, + ) + + return authenticationActivityRepository.findAllByUser(principal.user, pageRequest).map { it.toDto() } + } + + @GetMapping("/authentication-activity") + @PageableAsQueryParam + @PreAuthorize("hasRole('$ROLE_ADMIN')") + fun getAuthenticationActivity( + @RequestParam(name = "unpaged", required = false) unpaged: Boolean = false, + @Parameter(hidden = true) page: Pageable, + ): Page { + val sort = + if (page.sort.isSorted) page.sort + else Sort.by(Sort.Order.desc("dateTime")) + + val pageRequest = + if (unpaged) UnpagedSorted(sort) + else PageRequest.of( + page.pageNumber, + page.pageSize, + sort, + ) + + return authenticationActivityRepository.findAll(pageRequest).map { it.toDto() } + } + + @GetMapping("/{id}/authentication-activity/latest") + @PreAuthorize("hasRole('$ROLE_ADMIN') or #principal.user.id == #id") + fun getLatestAuthenticationActivityForUser( + @PathVariable id: String, + @AuthenticationPrincipal principal: KomgaPrincipal, + ): AuthenticationActivityDto = + userRepository.findByIdOrNull(id)?.let { user -> + authenticationActivityRepository.findMostRecentByUser(user)?.toDto() + ?: throw ResponseStatusException(HttpStatus.NOT_FOUND) + } ?: throw ResponseStatusException(HttpStatus.NOT_FOUND) +} \ No newline at end of file diff --git a/spec/functional_test/testers/java_armeria_spec.cr b/spec/functional_test/testers/java_armeria_spec.cr new file mode 100644 index 00000000..4638f715 --- /dev/null +++ b/spec/functional_test/testers/java_armeria_spec.cr @@ -0,0 +1,15 @@ +require "../func_spec.cr" + +extected_endpoints = [ + Endpoint.new("/graphql", "GET"), + Endpoint.new("/internal/l7check", "GET"), + Endpoint.new("/zipkin/config.json", "GET"), + Endpoint.new("/zipkin/api", "GET"), + Endpoint.new("/zipkin", "GET"), + Endpoint.new("/", "GET"), +] + +FunctionalTester.new("fixtures/java_armeria/", { + :techs => 1, + :endpoints => 6, +}, extected_endpoints).test_all diff --git a/spec/functional_test/testers/java_spring_spec.cr b/spec/functional_test/testers/java_spring_spec.cr index 27c7937a..dc801eba 100644 --- a/spec/functional_test/testers/java_spring_spec.cr +++ b/spec/functional_test/testers/java_spring_spec.cr @@ -20,7 +20,7 @@ extected_endpoints = [ Endpoint.new("/items/delete/{id}", "DELETE"), ] -FunctionalTester.new("fixtures/spring/", { +FunctionalTester.new("fixtures/java_spring/", { :techs => 1, :endpoints => 14, }, extected_endpoints).test_all diff --git a/spec/functional_test/testers/kotlin_spring_spec.cr b/spec/functional_test/testers/kotlin_spring_spec.cr new file mode 100644 index 00000000..9337b180 --- /dev/null +++ b/spec/functional_test/testers/kotlin_spring_spec.cr @@ -0,0 +1,19 @@ +require "../func_spec.cr" + +extected_endpoints = [ + Endpoint.new("/api/v2/users/me", "GET"), + Endpoint.new("/api/v2/users/me/password", "PATCH"), + Endpoint.new("/api/v2/users", "GET"), + Endpoint.new("/api/v2/users", "POST"), + Endpoint.new("/api/v2/users/{id}", "DELETE"), + Endpoint.new("/api/v2/users/{id}", "PATCH"), + Endpoint.new("/api/v2/users/{id}/password", "PATCH"), + Endpoint.new("/api/v2/users/me/authentication-activity", "GET"), + Endpoint.new("/api/v2/users/authentication-activity", "GET"), + Endpoint.new("/api/v2/users/{id}/authentication-activity/latest", "GET"), +] + +FunctionalTester.new("fixtures/kotlin_spring/", { + :techs => 1, + :endpoints => 10, +}, extected_endpoints).test_all diff --git a/spec/unit_test/detector/detect_java_armeria_spec.cr b/spec/unit_test/detector/detect_java_armeria_spec.cr new file mode 100644 index 00000000..5118cbdf --- /dev/null +++ b/spec/unit_test/detector/detect_java_armeria_spec.cr @@ -0,0 +1,13 @@ +require "../../../src/detector/detectors/*" + +describe "Detect Java Armeria" do + options = default_options() + instance = DetectorJavaArmeria.new options + + it "pom.xml" do + instance.detect("pom.xml", "com.linecorp.armeria").should eq(true) + end + it "build.gradle" do + instance.detect("build.gradle", "com.linecorp.armeria").should eq(true) + end +end diff --git a/src/analyzer/analyzer.cr b/src/analyzer/analyzer.cr index 491d9645..ec4e480d 100644 --- a/src/analyzer/analyzer.cr +++ b/src/analyzer/analyzer.cr @@ -5,6 +5,8 @@ def initialize_analyzers(logger : NoirLogger) analyzers["ruby_rails"] = ->analyzer_rails(Hash(Symbol, String)) analyzers["ruby_sinatra"] = ->analyzer_sinatra(Hash(Symbol, String)) analyzers["java_spring"] = ->analyzer_spring(Hash(Symbol, String)) + analyzers["kotlin_spring"] = ->analyzer_spring(Hash(Symbol, String)) + analyzers["java_armeria"] = ->analyzer_armeria(Hash(Symbol, String)) analyzers["php_pure"] = ->analyzer_php_pure(Hash(Symbol, String)) analyzers["go_echo"] = ->analyzer_go_echo(Hash(Symbol, String)) analyzers["go_gin"] = ->analyzer_go_gin(Hash(Symbol, String)) @@ -32,13 +34,16 @@ def analysis_endpoints(options : Hash(Symbol, String), techs, logger : NoirLogge analyzer = initialize_analyzers logger logger.info_sub "Analysis to #{techs.size} technologies" + if (techs.includes? "java_spring") && (techs.includes? "kotlin_spring") + techs.delete("kotlin_spring") + end + techs.each do |tech| if analyzer.has_key?(tech) if NoirTechs.similar_to_tech(options[:exclude_techs]).includes?(tech) logger.info_sub "Skipping #{tech} analysis" next end - result = result + analyzer[tech].call(options) end end diff --git a/src/analyzer/analyzers/analyzer_armeria.cr b/src/analyzer/analyzers/analyzer_armeria.cr new file mode 100644 index 00000000..875d4782 --- /dev/null +++ b/src/analyzer/analyzers/analyzer_armeria.cr @@ -0,0 +1,64 @@ +require "../../models/analyzer" + +class AnalyzerArmeria < Analyzer + REGEX_SERVER_CODE_BLOCK = /Server\s*\.builder\(\s*\)\s*\.[^;]*?build\(\)\s*\./ + REGEX_SERVICE_CODE = /\.service(If|Under|)?\([^;]+?\)/ + REGEX_ROUTE_CODE = /\.route\(\)\s*\.\s*(\w+)\s*\(([^\.]*)\)\./ + + def analyze + # Source Analysis + Dir.glob("#{@base_path}/**/*") do |path| + next if File.directory?(path) + + url = @url + if File.exists?(path) && (path.ends_with?(".java") || path.ends_with?(".kt")) + content = File.read(path, encoding: "utf-8", invalid: :skip) + content.scan(REGEX_SERVER_CODE_BLOCK) do |server_codeblcok_match| + server_codeblock = server_codeblcok_match[0] + + server_codeblock.scan(REGEX_SERVICE_CODE) do |service_code_match| + next if service_code_match.size != 2 + endpoint_param_index = 0 + if service_code_match[1] == "If" + endpoint_param_index = 1 + end + + service_code = service_code_match[0] + parameter_code = service_code.split("(")[1] + split_params = parameter_code.split(",") + next if split_params.size <= endpoint_param_index + endpoint = split_params[endpoint_param_index].strip + + endpoint = endpoint[1..-2] + @result << Endpoint.new("#{url}#{endpoint}", "GET") + end + + server_codeblock.scan(REGEX_ROUTE_CODE) do |route_code_match| + next if route_code_match.size != 3 + method = route_code_match[1].upcase + if method == "PATH" + method = "GET" + end + + next if !["GET", "POST", "DELETE", "PUT", "PATCH", "HEAD", "OPTIONS"].includes?(method) + + endpoint = route_code_match[2].split(")")[0].strip + next if endpoint[0] != endpoint[-1] + next if endpoint[0] != '"' + + endpoint = endpoint[1..-2] + @result << Endpoint.new("#{url}#{endpoint}", method) + end + end + end + end + Fiber.yield + + @result + end +end + +def analyzer_armeria(options : Hash(Symbol, String)) + instance = AnalyzerArmeria.new(options) + instance.analyze +end diff --git a/src/analyzer/analyzers/analyzer_spring.cr b/src/analyzer/analyzers/analyzer_spring.cr index 5dc9408b..a00db771 100644 --- a/src/analyzer/analyzers/analyzer_spring.cr +++ b/src/analyzer/analyzers/analyzer_spring.cr @@ -11,7 +11,7 @@ class AnalyzerSpring < Analyzer next if File.directory?(path) url = @url - if File.exists?(path) && path.ends_with?(".java") + if File.exists?(path) && (path.ends_with?(".java") || path.ends_with?(".kt")) content = File.read(path, encoding: "utf-8", invalid: :skip) # Spring MVC @@ -75,16 +75,16 @@ class AnalyzerSpring < Analyzer @result << Endpoint.new("#{url}#{mapping_path}", "GET") end end + end - # Reactive Router - content.scan(REGEX_ROUTER_CODE_BLOCK) do |route_code| - method_code = route_code[0] - method_code.scan(REGEX_ROUTE_CODE_LINE) do |match| - next if match.size != 4 - method = match[2] - endpoint = match[3].gsub(/\n/, "") - @result << Endpoint.new("#{url}#{endpoint}", method) - end + # Reactive Router + content.scan(REGEX_ROUTER_CODE_BLOCK) do |route_code| + method_code = route_code[0] + method_code.scan(REGEX_ROUTE_CODE_LINE) do |match| + next if match.size != 4 + method = match[2] + endpoint = match[3].gsub(/\n/, "") + @result << Endpoint.new("#{url}#{endpoint}", method) end end end @@ -102,6 +102,7 @@ class AnalyzerSpring < Analyzer line = splited_line[1].gsub(/"|\)| /, "").gsub(/\s/, "").strip if line.size > 0 if line[0].to_s == "/" + line = line.split(/,\w+=/)[0] paths << line else if is_bracket(line) diff --git a/src/detector/detector.cr b/src/detector/detector.cr index 684a81fa..d99315fa 100644 --- a/src/detector/detector.cr +++ b/src/detector/detector.cr @@ -16,7 +16,7 @@ def detect_techs(base_path : String, options : Hash(Symbol, String), logger : No DetectorCrystalKemal, DetectorGoEcho, DetectorJavaJsp, DetectorJavaSpring, DetectorJsExpress, DetectorPhpPure, DetectorPythonDjango, DetectorPythonFlask, DetectorRubyRails, DetectorRubySinatra, DetectorOas2, DetectorOas3, DetectorRAML, - DetectorGoGin, + DetectorGoGin, DetectorKotlinSpring, DetectorJavaArmeria, ]) Dir.glob("#{base_path}/**/*") do |file| spawn do diff --git a/src/detector/detectors/java_armeria.cr b/src/detector/detectors/java_armeria.cr new file mode 100644 index 00000000..773f8d16 --- /dev/null +++ b/src/detector/detectors/java_armeria.cr @@ -0,0 +1,18 @@ +require "../../models/detector" + +class DetectorJavaArmeria < Detector + def detect(filename : String, file_contents : String) : Bool + if ( + (filename.includes? "pom.xml") || (filename.includes? "build.gradle") || + (filename.includes? "build.gradle.kts") || (filename.includes? "settings.gradle.kts") + ) && (file_contents.includes? "com.linecorp.armeria") + true + else + false + end + end + + def set_name + @name = "java_armeria" + end +end diff --git a/src/detector/detectors/java_spring.cr b/src/detector/detectors/java_spring.cr index 652d4bae..766e635b 100644 --- a/src/detector/detectors/java_spring.cr +++ b/src/detector/detectors/java_spring.cr @@ -2,7 +2,9 @@ require "../../models/detector" class DetectorJavaSpring < Detector def detect(filename : String, file_contents : String) : Bool - if ((filename.includes? "pom.xml") || filename.includes? "build.gradle") && (file_contents.includes? "org.springframework") + if ( + (filename.includes? "pom.xml") || (filename.ends_with? "build.gradle") + ) && (file_contents.includes? "org.springframework") true else false diff --git a/src/detector/detectors/kotlin_spring.cr b/src/detector/detectors/kotlin_spring.cr new file mode 100644 index 00000000..3ea4cffc --- /dev/null +++ b/src/detector/detectors/kotlin_spring.cr @@ -0,0 +1,15 @@ +require "../../models/detector" + +class DetectorKotlinSpring < Detector + def detect(filename : String, file_contents : String) : Bool + if (filename.ends_with? "build.gradle.kts") && (file_contents.includes? "org.springframework") + true + else + false + end + end + + def set_name + @name = "kotlin_spring" + end +end diff --git a/src/techs/techs.cr b/src/techs/techs.cr index 209475a8..cc6521c2 100644 --- a/src/techs/techs.cr +++ b/src/techs/techs.cr @@ -25,6 +25,16 @@ module NoirTechs :framework => "Spring", :similar => ["spring", "java-spring", "java_spring"], }, + :java_armeria => { + :language => "Java", + :framework => "Armeria", + :similar => ["armeria", "java-armeria", "java_armeria"], + }, + :kotlin_spring => { + :language => "Kotlin", + :framework => "Spring", + :similar => ["spring", "kotlin-spring", "kotlin_spring"], + }, :js_express => { :language => "JavaScript", :framework => "Express", From 3238e1022454ea18471b62ca24c5e43bf8201173 Mon Sep 17 00:00:00 2001 From: ksg97031 Date: Thu, 7 Sep 2023 21:19:52 +0900 Subject: [PATCH 3/7] Fixed parsing error in spring analyzer Signed-off-by: ksg97031 --- .../fixtures/java_spring/src/ItemController.java | 4 ++++ spec/functional_test/testers/java_spring_spec.cr | 3 ++- spec/unit_test/analyzer/analyzer_spring_spec.cr | 3 +++ src/analyzer/analyzers/analyzer_spring.cr | 7 ++++++- 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/spec/functional_test/fixtures/java_spring/src/ItemController.java b/spec/functional_test/fixtures/java_spring/src/ItemController.java index e58b7e7f..88ebfb54 100644 --- a/spec/functional_test/fixtures/java_spring/src/ItemController.java +++ b/spec/functional_test/fixtures/java_spring/src/ItemController.java @@ -19,4 +19,8 @@ public Item updateItem(@PathVariable Long id, @RequestBody Item item) { @DeleteMapping("/delete/{id}") public void deleteItem(@PathVariable Long id) { } + + @GetMapping("/json/{id}", produces = [MediaType.APPLICATION_JSON_VALUE]) + public void getItemJson(){ + } } \ No newline at end of file diff --git a/spec/functional_test/testers/java_spring_spec.cr b/spec/functional_test/testers/java_spring_spec.cr index dc801eba..0ee49789 100644 --- a/spec/functional_test/testers/java_spring_spec.cr +++ b/spec/functional_test/testers/java_spring_spec.cr @@ -15,6 +15,7 @@ extected_endpoints = [ Endpoint.new("/quotes/0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-_.~", "GET"), # ItemController.java Endpoint.new("/items/{id}", "GET"), + Endpoint.new("/items/json/{id}", "GET"), Endpoint.new("/items", "POST"), Endpoint.new("/items/update/{id}", "PUT"), Endpoint.new("/items/delete/{id}", "DELETE"), @@ -22,5 +23,5 @@ extected_endpoints = [ FunctionalTester.new("fixtures/java_spring/", { :techs => 1, - :endpoints => 14, + :endpoints => 15, }, extected_endpoints).test_all diff --git a/spec/unit_test/analyzer/analyzer_spring_spec.cr b/spec/unit_test/analyzer/analyzer_spring_spec.cr index 893926ac..39cd3fa9 100644 --- a/spec/unit_test/analyzer/analyzer_spring_spec.cr +++ b/spec/unit_test/analyzer/analyzer_spring_spec.cr @@ -53,6 +53,9 @@ describe "mapping_to_path" do it "mapping_to_path - requestmapping style5" do instance.mapping_to_path("@RequestMapping(method = RequestMethod.GET)").should eq([""]) end + it "mapping_to_path - requestmapping style6" do + instance.mapping_to_path("@RequestMapping(\"/abcd\", produces=[MediaType.APPLICATION_JSON_VALUE])").should eq(["/abcd"]) + end end describe "utils func" do diff --git a/src/analyzer/analyzers/analyzer_spring.cr b/src/analyzer/analyzers/analyzer_spring.cr index a00db771..9a237b8e 100644 --- a/src/analyzer/analyzers/analyzer_spring.cr +++ b/src/analyzer/analyzers/analyzer_spring.cr @@ -102,7 +102,12 @@ class AnalyzerSpring < Analyzer line = splited_line[1].gsub(/"|\)| /, "").gsub(/\s/, "").strip if line.size > 0 if line[0].to_s == "/" - line = line.split(/,\w+=/)[0] + attribute_index = line.index(/,(\w)+=/) + if !attribute_index.nil? + attribute_index -= 1 + line = line[0..attribute_index] + end + paths << line else if is_bracket(line) From a573d043b7e4047fa98569b55758663118647878 Mon Sep 17 00:00:00 2001 From: ksg97031 Date: Sun, 10 Sep 2023 01:12:17 +0900 Subject: [PATCH 4/7] Import Django Analyzer(method, param, header) Signed-off-by: ksg97031 --- README.md | 2 +- .../functional_test/fixtures/django/README.md | 3 + .../fixtures/django/accounts/urls.py | 18 - .../fixtures/django/blog/__init__.py | 0 .../fixtures/django/blog/urls.py | 70 +++ .../fixtures/django/blog/views.py | 395 +++++++++++++ .../fixtures/django/djangoblog/__init__.py | 0 .../fixtures/django/djangoblog/settings.py | 81 +++ .../fixtures/django/djangoblog/urls.py | 47 ++ .../django/djangoproject/settings/common.py | 5 - .../fixtures/django/djangoproject/urls/www.py | 13 - .../functional_test/fixtures/django/manage.py | 30 +- .../testers/python_django_spec.cr | 29 +- .../analyzer/analyzer_django_spec.cr | 23 - src/analyzer/analyzers/analyzer_django.cr | 530 ++++++++++++++++-- 15 files changed, 1118 insertions(+), 128 deletions(-) create mode 100644 spec/functional_test/fixtures/django/README.md delete mode 100644 spec/functional_test/fixtures/django/accounts/urls.py create mode 100644 spec/functional_test/fixtures/django/blog/__init__.py create mode 100644 spec/functional_test/fixtures/django/blog/urls.py create mode 100644 spec/functional_test/fixtures/django/blog/views.py create mode 100644 spec/functional_test/fixtures/django/djangoblog/__init__.py create mode 100644 spec/functional_test/fixtures/django/djangoblog/settings.py create mode 100644 spec/functional_test/fixtures/django/djangoblog/urls.py delete mode 100644 spec/functional_test/fixtures/django/djangoproject/settings/common.py delete mode 100644 spec/functional_test/fixtures/django/djangoproject/urls/www.py delete mode 100644 spec/unit_test/analyzer/analyzer_django_spec.cr diff --git a/README.md b/README.md index d32d20d5..00acf97d 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ |----------|-----------|-----|--------|-------|--------|----| | Go | Echo | ✅ | ✅ | ✅ | ✅ | X | | Go | Gin | ✅ | ✅ | ✅ | ✅ | X | -| Python | Django | ✅ | X | X | X | X | +| Python | Django | ✅ | ✅ | ✅ | ✅ | X | | Python | Flask | ✅ | X | X | X | X | | Ruby | Rails | ✅ | ✅ | ✅ | ✅ | X | | Ruby | Sinatra | ✅ | ✅ | ✅ | ✅ | X | diff --git a/spec/functional_test/fixtures/django/README.md b/spec/functional_test/fixtures/django/README.md new file mode 100644 index 00000000..e6d2a4de --- /dev/null +++ b/spec/functional_test/fixtures/django/README.md @@ -0,0 +1,3 @@ +## Notes + +This project uses a portion of the source code from [DjangoBlog](https://github.com/liangliangyy/DjangoBlog) for testing purposes. (Djangoblog is released under the MIT License) \ No newline at end of file diff --git a/spec/functional_test/fixtures/django/accounts/urls.py b/spec/functional_test/fixtures/django/accounts/urls.py deleted file mode 100644 index 3c4d3153..00000000 --- a/spec/functional_test/fixtures/django/accounts/urls.py +++ /dev/null @@ -1,18 +0,0 @@ -from django.urls import path -from registration.backends.default.views import RegistrationView -from registration.forms import RegistrationFormUniqueEmail - -from . import views as account_views - -urlpatterns = [ - path( - "register/", - RegistrationView.as_view(form_class=RegistrationFormUniqueEmail), - name="registration_register", - ), - path( - "edit/", - account_views.edit_profile, - name="edit_profile", - ), -] diff --git a/spec/functional_test/fixtures/django/blog/__init__.py b/spec/functional_test/fixtures/django/blog/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/spec/functional_test/fixtures/django/blog/urls.py b/spec/functional_test/fixtures/django/blog/urls.py new file mode 100644 index 00000000..bd75633c --- /dev/null +++ b/spec/functional_test/fixtures/django/blog/urls.py @@ -0,0 +1,70 @@ +from django.urls import path +from django.views.decorators.cache import cache_page + +from . import views + +app_name = "blog" +urlpatterns = [ + path( + r'', + views.IndexView.as_view(), + name='index'), + path( + r'page//', + views.IndexView.as_view(), + name='index_page'), + path( + r'article////.html', + views.ArticleDetailView.as_view(), + name='detailbyid'), + path( + r'category/.html', + views.CategoryDetailView.as_view(), + name='category_detail'), + path( + r'category//.html', + views.CategoryDetailView.as_view(), + name='category_detail_page'), + path( + r'author/.html', + views.AuthorDetailView.as_view(), + name='author_detail'), + path( + r'author//.html', + views.AuthorDetailView.as_view(), + name='author_detail_page'), + path( + r'tag/.html', + views.TagDetailView.as_view(), + name='tag_detail'), + path( + r'tag//.html', + views.TagDetailView.as_view(), + name='tag_detail_page'), + path( + 'archives.html', + cache_page( + 60 * 60)( + views.ArchivesView.as_view()), + name='archives'), + path( + 'links.html', + views.LinkListView.as_view(), + name='links'), + path( + r'upload', + views.fileupload, + name='upload'), + path( + r'not_found', + views.page_not_found_view, + name='page_not_found_view'), + path( + r'test', + views.test, + name='test'), + path( + r'delete_test', + views.delete_test, + name='delete_test'), +] diff --git a/spec/functional_test/fixtures/django/blog/views.py b/spec/functional_test/fixtures/django/blog/views.py new file mode 100644 index 00000000..8821314b --- /dev/null +++ b/spec/functional_test/fixtures/django/blog/views.py @@ -0,0 +1,395 @@ +import logging +import os +import uuid + +from django.conf import settings +from django.core.paginator import Paginator +from django.http import HttpResponse, HttpResponseForbidden +from django.shortcuts import get_object_or_404 +from django.shortcuts import render +from django.templatetags.static import static +from django.utils import timezone +from django.views.decorators.csrf import csrf_exempt +from django.views.generic.detail import DetailView +from django.views.generic.list import ListView +from haystack.views import SearchView + +from blog.models import Article, Category, LinkShowType, Links, Tag +from comments.forms import CommentForm +from djangoblog.utils import cache, get_blog_setting, get_sha256 + +logger = logging.getLogger(__name__) + + +class ArticleListView(ListView): + # template_name属性用于指定使用哪个模板进行渲染 + template_name = 'blog/article_index.html' + + # context_object_name属性用于给上下文变量取名（在模板中使用该名字） + context_object_name = 'article_list' + + # 页面类型，分类目录或标签列表等 + page_type = '' + paginate_by = settings.PAGINATE_BY + page_kwarg = 'page' + link_type = LinkShowType.L + + def get_view_cache_key(self): + return self.request.get['pages'] + + @property + def page_number(self): + page_kwarg = self.page_kwarg + page = self.kwargs.get( + page_kwarg) or self.request.GET.get(page_kwarg) or 1 + return page + + def get_queryset_cache_key(self): + """ + 子类重写.获得queryset的缓存key + """ + raise NotImplementedError() + + def get_queryset_data(self): + """ + 子类重写.获取queryset的数据 + """ + raise NotImplementedError() + + def get_queryset_from_cache(self, cache_key): + ''' + 缓存页面数据 + :param cache_key: 缓存key + :return: + ''' + value = cache.get(cache_key) + if value: + logger.info('get view cache.key:{key}'.format(key=cache_key)) + return value + else: + article_list = self.get_queryset_data() + cache.set(cache_key, article_list) + logger.info('set view cache.key:{key}'.format(key=cache_key)) + return article_list + + def get_queryset(self): + ''' + 重写默认，从缓存获取数据 + :return: + ''' + key = self.get_queryset_cache_key() + value = self.get_queryset_from_cache(key) + return value + + def get_context_data(self, **kwargs): + kwargs['linktype'] = self.link_type + return super(ArticleListView, self).get_context_data(**kwargs) + + +class IndexView(ArticleListView): + ''' + 首页 + ''' + # 友情链接类型 + link_type = LinkShowType.I + + def get_queryset_data(self): + article_list = Article.objects.filter(type='a', status='p') + return article_list + + def get_queryset_cache_key(self): + cache_key = 'index_{page}'.format(page=self.page_number) + return cache_key + + +class ArticleDetailView(DetailView): + ''' + 文章详情页面 + ''' + template_name = 'blog/article_detail.html' + model = Article + pk_url_kwarg = 'article_id' + context_object_name = "article" + + def get_object(self, queryset=None): + obj = super(ArticleDetailView, self).get_object() + obj.viewed() + self.object = obj + return obj + + def get_context_data(self, **kwargs): + comment_form = CommentForm() + + article_comments = self.object.comment_list() + parent_comments = article_comments.filter(parent_comment=None) + blog_setting = get_blog_setting() + paginator = Paginator(parent_comments, blog_setting.article_comment_count) + page = self.request.GET.get('comment_page', '1') + if not page.isnumeric(): + page = 1 + else: + page = int(page) + if page < 1: + page = 1 + if page > paginator.num_pages: + page = paginator.num_pages + + p_comments = paginator.page(page) + next_page = p_comments.next_page_number() if p_comments.has_next() else None + prev_page = p_comments.previous_page_number() if p_comments.has_previous() else None + + if next_page: + kwargs[ + 'comment_next_page_url'] = self.object.get_absolute_url() + f'?comment_page={next_page}#commentlist-container' + if prev_page: + kwargs[ + 'comment_prev_page_url'] = self.object.get_absolute_url() + f'?comment_page={prev_page}#commentlist-container' + kwargs['form'] = comment_form + kwargs['article_comments'] = article_comments + kwargs['p_comments'] = p_comments + kwargs['comment_count'] = len( + article_comments) if article_comments else 0 + + kwargs['next_article'] = self.object.next_article + kwargs['prev_article'] = self.object.prev_article + + return super(ArticleDetailView, self).get_context_data(**kwargs) + + +class CategoryDetailView(ArticleListView): + ''' + 分类目录列表 + ''' + page_type = "分类目录归档" + + def get_queryset_data(self): + slug = self.kwargs['category_name'] + category = get_object_or_404(Category, slug=slug) + + categoryname = category.name + self.categoryname = categoryname + categorynames = list( + map(lambda c: c.name, category.get_sub_categorys())) + article_list = Article.objects.filter( + category__name__in=categorynames, status='p') + return article_list + + def get_queryset_cache_key(self): + slug = self.kwargs['category_name'] + category = get_object_or_404(Category, slug=slug) + categoryname = category.name + self.categoryname = categoryname + cache_key = 'category_list_{categoryname}_{page}'.format( + categoryname=categoryname, page=self.page_number) + return cache_key + + def get_context_data(self, **kwargs): + + categoryname = self.categoryname + try: + categoryname = categoryname.split('/')[-1] + except BaseException: + pass + kwargs['page_type'] = CategoryDetailView.page_type + kwargs['tag_name'] = categoryname + return super(CategoryDetailView, self).get_context_data(**kwargs) + + +class AuthorDetailView(ArticleListView): + ''' + 作者详情页 + ''' + page_type = '作者文章归档' + + def get_queryset_cache_key(self): + from uuslug import slugify + author_name = slugify(self.kwargs['author_name']) + cache_key = 'author_{author_name}_{page}'.format( + author_name=author_name, page=self.page_number) + return cache_key + + def get_queryset_data(self): + author_name = self.kwargs['author_name'] + article_list = Article.objects.filter( + author__username=author_name, type='a', status='p') + return article_list + + def get_context_data(self, **kwargs): + author_name = self.kwargs['author_name'] + kwargs['page_type'] = AuthorDetailView.page_type + kwargs['tag_name'] = author_name + return super(AuthorDetailView, self).get_context_data(**kwargs) + + +class TagDetailView(ArticleListView): + ''' + 标签列表页面 + ''' + page_type = '分类标签归档' + + def get_queryset_data(self): + slug = self.kwargs['tag_name'] + tag = get_object_or_404(Tag, slug=slug) + tag_name = tag.name + self.name = tag_name + article_list = Article.objects.filter( + tags__name=tag_name, type='a', status='p') + return article_list + + def get_queryset_cache_key(self): + slug = self.kwargs['tag_name'] + tag = get_object_or_404(Tag, slug=slug) + tag_name = tag.name + self.name = tag_name + cache_key = 'tag_{tag_name}_{page}'.format( + tag_name=tag_name, page=self.page_number) + return cache_key + + def get_context_data(self, **kwargs): + # tag_name = self.kwargs['tag_name'] + tag_name = self.name + kwargs['page_type'] = TagDetailView.page_type + kwargs['tag_name'] = tag_name + return super(TagDetailView, self).get_context_data(**kwargs) + + +class ArchivesView(ArticleListView): + ''' + 文章归档页面 + ''' + page_type = '文章归档' + paginate_by = None + page_kwarg = None + template_name = 'blog/article_archives.html' + + def get_queryset_data(self): + return Article.objects.filter(status='p').all() + + def get_queryset_cache_key(self): + cache_key = 'archives' + return cache_key + + +class LinkListView(ListView): + model = Links + template_name = 'blog/links_list.html' + + def get_queryset(self): + return Links.objects.filter(is_enable=True) + + +class EsSearchView(SearchView): + def get_context(self): + paginator, page = self.build_page() + context = { + "query": self.query, + "form": self.form, + "page": page, + "paginator": paginator, + "suggestion": None, + } + if hasattr(self.results, "query") and self.results.query.backend.include_spelling: + context["suggestion"] = self.results.query.get_spelling_suggestion() + context.update(self.extra_context()) + + return context + + +@csrf_exempt +def fileupload(request): + """ + 该方法需自己写调用端来上传图片，该方法仅提供图床功能 + :param request: + :return: + """ + status = request.META.get('HTTP_X_FORWARDED_FOR') == '127.0.0.1' + status = status and request.META.get('X_REAL_IP') == '127.0.0.1' + + if request.method == 'POST': + sign = request.GET.get('sign', None) + if not sign: + return HttpResponseForbidden() + if not sign == get_sha256(get_sha256(settings.SECRET_KEY)): + return HttpResponseForbidden() + + if not status: + return "is not localhost" + + response = [] + for filename in request.FILES: + timestr = timezone.now().strftime('%Y/%m/%d') + imgextensions = ['jpg', 'png', 'jpeg', 'bmp'] + fname = u''.join(str(filename)) + isimage = len([i for i in imgextensions if fname.find(i) >= 0]) > 0 + base_dir = os.path.join(settings.STATICFILES, "files" if not isimage else "image", timestr) + if not os.path.exists(base_dir): + os.makedirs(base_dir) + savepath = os.path.normpath(os.path.join(base_dir, f"{uuid.uuid4().hex}{os.path.splitext(filename)[-1]}")) + if not savepath.startswith(base_dir): + return HttpResponse("only for post") + with open(savepath, 'wb+') as wfile: + for chunk in request.FILES[filename].chunks(): + wfile.write(chunk) + if isimage: + from PIL import Image + image = Image.open(savepath) + image.save(savepath, quality=20, optimize=True) + url = static(savepath) + response.append(url) + return HttpResponse(response) + + else: + return HttpResponse("only for post") + + +def page_not_found_view(request, + exception=None, + template_name='blog/error_page.html'): + if exception: + logger.error(exception) + url = request.get_full_path() + + dummpy_cookie = '' + if request.COOKIES['app_type'] == 'ksg97031': + dummpy_cookie = 'ksg97031' + + + return render(request, + template_name, + {'message': '哎呀，您访问的地址 ' + url + ' 是一个未知的地方。请点击首页看看别的？', + 'statuscode': '404'}, + status=404) + +def test(request): + if request.method == 'POST': + return request.data.get('test_param', 'no params') + if request.method == 'PUT': + return request.data.get('test_param', 'no params') + if request.method == 'PATCH': + return request.data.get('test_param', 'no params') + + return "test" + +def delete_test(request): + if request.method == 'DELETE': + return 'delete' + + return "test" + +def server_error_view(request, template_name='blog/error_page.html'): + return render(request, + template_name, + {'message': '哎呀，出错了，我已经收集到了错误信息，之后会抓紧抢修，请点击首页看看别的？', + 'statuscode': '500'}, + status=500) + + +def permission_denied_view( + request, + exception, + template_name='blog/error_page.html'): + if exception: + logger.error(exception) + return render( + request, template_name, { + 'message': '哎呀，您没有权限访问此页面，请点击首页看看别的？', 'statuscode': '403'}, status=403) diff --git a/spec/functional_test/fixtures/django/djangoblog/__init__.py b/spec/functional_test/fixtures/django/djangoblog/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/spec/functional_test/fixtures/django/djangoblog/settings.py b/spec/functional_test/fixtures/django/djangoblog/settings.py new file mode 100644 index 00000000..e64f3a24 --- /dev/null +++ b/spec/functional_test/fixtures/django/djangoblog/settings.py @@ -0,0 +1,81 @@ +""" +Django settings for djangoblog project. + +Generated by 'django-admin startproject' using Django 1.10.2. + +For more information on this file, see +https://docs.djangoproject.com/en/1.10/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/1.10/ref/settings/ +""" +import os +import sys + + +def env_to_bool(env, default): + str_val = os.environ.get(env) + return default if str_val is None else str_val == 'True' + + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) + +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = os.environ.get( + 'DJANGO_SECRET_KEY') or 'n9ceqv38)#&mwuat@(mjb_p%em$e8$qyr#fw9ot!=ba6lijx-6' +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = env_to_bool('DJANGO_DEBUG', True) +# DEBUG = False +TESTING = len(sys.argv) > 1 and sys.argv[1] == 'test' + +# ALLOWED_HOSTS = [] +ALLOWED_HOSTS = ['*', '127.0.0.1', 'example.com'] +# django 4.0新增配置 +CSRF_TRUSTED_ORIGINS = ['http://example.com'] +# Application definition + + +INSTALLED_APPS = [ + # 'django.contrib.admin', + 'django.contrib.admin.apps.SimpleAdminConfig', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'django.contrib.sites', + 'django.contrib.sitemaps', + 'mdeditor', + 'haystack', + 'blog', + 'accounts', + 'comments', + 'oauth', + 'servermanager', + 'owntracks', + 'compressor' +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.gzip.GZipMiddleware', + # 'django.middleware.cache.UpdateCacheMiddleware', + 'django.middleware.common.CommonMiddleware', + # 'django.middleware.cache.FetchFromCacheMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'django.middleware.http.ConditionalGetMiddleware', + 'blog.middleware.OnlineMiddleware' +] + +ROOT_URLCONF = 'djangoblog.urls' + +# [REDACTED] diff --git a/spec/functional_test/fixtures/django/djangoblog/urls.py b/spec/functional_test/fixtures/django/djangoblog/urls.py new file mode 100644 index 00000000..cfd94b10 --- /dev/null +++ b/spec/functional_test/fixtures/django/djangoblog/urls.py @@ -0,0 +1,47 @@ +"""djangoblog URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/1.10/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: url(r'^$', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.conf.urls import url, include + 2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) +""" +from django.conf import settings +from django.conf.urls.static import static +from django.contrib.sitemaps.views import sitemap +from django.urls import include +from django.urls import re_path +from haystack.views import search_view_factory + +from blog.views import EsSearchView +from djangoblog.admin_site import admin_site +from djangoblog.elasticsearch_backend import ElasticSearchModelSearchForm +from djangoblog.feeds import DjangoBlogFeed +from djangoblog.sitemap import ArticleSiteMap, CategorySiteMap, StaticViewSitemap, TagSiteMap, UserSiteMap + +sitemaps = { + + 'blog': ArticleSiteMap, + 'Category': CategorySiteMap, + 'Tag': TagSiteMap, + 'User': UserSiteMap, + 'static': StaticViewSitemap +} + +handler404 = 'blog.views.page_not_found_view' +handler500 = 'blog.views.server_error_view' +handle403 = 'blog.views.permission_denied_view' +urlpatterns = [ + re_path(r'', include('blog.urls', namespace='blog')), + # [REDACTED] + ] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) +if settings.DEBUG: + urlpatterns += static(settings.MEDIA_URL, + document_root=settings.MEDIA_ROOT) diff --git a/spec/functional_test/fixtures/django/djangoproject/settings/common.py b/spec/functional_test/fixtures/django/djangoproject/settings/common.py deleted file mode 100644 index 941129e2..00000000 --- a/spec/functional_test/fixtures/django/djangoproject/settings/common.py +++ /dev/null @@ -1,5 +0,0 @@ -# [REDACTED] - -ROOT_URLCONF = "djangoproject.urls.www" - -# [REDACTED] \ No newline at end of file diff --git a/spec/functional_test/fixtures/django/djangoproject/urls/www.py b/spec/functional_test/fixtures/django/djangoproject/urls/www.py deleted file mode 100644 index 7df28b6f..00000000 --- a/spec/functional_test/fixtures/django/djangoproject/urls/www.py +++ /dev/null @@ -1,13 +0,0 @@ -from django.urls import include, path -from django.views.generic import RedirectView, TemplateView - -urlpatterns = [ - path( - "start/overview/", - TemplateView.as_view(template_name="overview.html"), - name="overview", - ), - path("overview/", RedirectView.as_view(url="/start/overview/", permanent=False)), - # include - path("accounts/", include("accounts.urls")), -] diff --git a/spec/functional_test/fixtures/django/manage.py b/spec/functional_test/fixtures/django/manage.py index 6d406775..919ba740 100755 --- a/spec/functional_test/fixtures/django/manage.py +++ b/spec/functional_test/fixtures/django/manage.py @@ -1,22 +1,22 @@ #!/usr/bin/env python -"""Django's command-line utility for administrative tasks.""" import os import sys - -def main(): - """Run administrative tasks.""" - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "djangoproject.settings.dev") +if __name__ == "__main__": + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "djangoblog.settings") try: from django.core.management import execute_from_command_line - except ImportError as exc: - raise ImportError( - "Couldn't import Django. Are you sure it's installed and " - "available on your PYTHONPATH environment variable? Did you " - "forget to activate a virtual environment?" - ) from exc + except ImportError: + # The above import may fail for some other reason. Ensure that the + # issue is really that Django is missing to avoid masking other + # exceptions on Python 2. + try: + import django + except ImportError: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) + raise execute_from_command_line(sys.argv) - - -if __name__ == "__main__": - main() diff --git a/spec/functional_test/testers/python_django_spec.cr b/spec/functional_test/testers/python_django_spec.cr index 39739b7d..0406ec1f 100644 --- a/spec/functional_test/testers/python_django_spec.cr +++ b/spec/functional_test/testers/python_django_spec.cr @@ -1,16 +1,29 @@ require "../func_spec.cr" extected_endpoints = [ - # djangoproject/urls/www.py - Endpoint.new("/start/overview/", "GET"), - Endpoint.new("/overview/", "GET"), - # accounts/urls.py - Endpoint.new("/accounts/register/", "GET"), - Endpoint.new("/accounts/edit/", "GET"), - Endpoint.new("/accounts/", "GET"), + Endpoint.new("/", "GET"), + Endpoint.new("/page//", "GET"), + Endpoint.new("/article////.html", "GET", [Param.new("comment_page", "", "query")]), + Endpoint.new("/category/.html", "GET"), + Endpoint.new("/category//.html", "GET"), + Endpoint.new("/author/.html", "GET"), + Endpoint.new("/author//.html", "GET"), + Endpoint.new("/tag/.html", "GET"), + Endpoint.new("/tag//.html", "GET"), + Endpoint.new("/archives.html", "GET"), + Endpoint.new("/links.html", "GET"), + Endpoint.new("/upload", "GET", [Param.new("sign", "", "query"), Param.new("sign", "", "query"), Param.new("X_FORWARDED_FOR", "", "header"), Param.new("X_REAL_IP", "", "header")]), + Endpoint.new("/upload", "POST", [Param.new("sign", "", "query"), Param.new("X_FORWARDED_FOR", "", "header"), Param.new("X_REAL_IP", "", "header")]), + Endpoint.new("/not_found", "GET", [Param.new("Cookie['app_type']", "", "header")]), + Endpoint.new("/test", "GET", [Param.new("test_param", "", "form")]), + Endpoint.new("/test", "POST", [Param.new("test_param", "", "form")]), + Endpoint.new("/test", "PUT", [Param.new("test_param", "", "form")]), + Endpoint.new("/test", "PATCH", [Param.new("test_param", "", "form")]), + Endpoint.new("/delete_test", "GET"), + Endpoint.new("/delete_test", "DELETE"), ] FunctionalTester.new("fixtures/django/", { :techs => 1, - :endpoints => 5, + :endpoints => 20, }, extected_endpoints).test_all diff --git a/spec/unit_test/analyzer/analyzer_django_spec.cr b/spec/unit_test/analyzer/analyzer_django_spec.cr deleted file mode 100644 index 7440a1f0..00000000 --- a/spec/unit_test/analyzer/analyzer_django_spec.cr +++ /dev/null @@ -1,23 +0,0 @@ -require "../../../src/analyzer/analyzers/analyzer_django.cr" -require "../../../src/options" - -describe "mapping_to_path" do - options = default_options() - instance = AnalyzerDjango.new(options) - - it "mapping_to_path - code style1" do - instance.mapping_to_path("path('home/', views.home_view, name='home'),").should eq(["/home/"]) - end - - it "mapping_to_path - code style2" do - instance.mapping_to_path("path('articles//', views.article_detail_view, name='article_detail'),").should eq(["/articles//"]) - end - - it "mapping_to_path - code style3 (regex)" do - instance.mapping_to_path("re_path(r'^archive/(?Pd{4})/$', views.archive_year_view, name='archive_year'),").should eq(["/archive/(?Pd{4})/"]) - end - - it "mapping_to_path - code style4 (register)" do - instance.mapping_to_path("router.register(r'articles', ArticleViewSet)").should eq(["/articles"]) - end -end diff --git a/src/analyzer/analyzers/analyzer_django.cr b/src/analyzer/analyzers/analyzer_django.cr index 18a4f517..ce8144ea 100644 --- a/src/analyzer/analyzers/analyzer_django.cr +++ b/src/analyzer/analyzers/analyzer_django.cr @@ -2,10 +2,25 @@ require "../../models/analyzer" require "json" class AnalyzerDjango < Analyzer - REGEX_ROOT_URLCONF = /\s*ROOT_URLCONF\s*=\s*r?['"]([^'"\\]*)['"]/ - REGEX_URL_PATTERNS = /urlpatterns\s*=\s*\[(.*)\]/m - REGEX_URL_MAPPING = /(?:url|path|register)\s*\(\s*r?['"]([^"']*)['"]\s*,\s*([^),]*)/ - REGEX_INCLUDE_URLS = /include\s*\(\s*r?['"]([^'"\\]*)['"]/ + @django_base_path : String = "" + REGEX_ROOT_URLCONF = /\s*ROOT_URLCONF\s*=\s*r?['"]([^'"\\]*)['"]/ + REGEX_ROUTE_MAPPING = /(?:url|path|register)\s*\(\s*r?['"]([^"']*)['"][^,]*,\s*([^),]*)/ + REGEX_INCLUDE_URLS = /include\s*\(\s*r?['"]([^'"\\]*)['"]/ + INDENT_SPACE_SIZE = 4 # Different indentation sizes can result in code analysis being disregarded + HTTP_METHOD_NAMES = ["get", "post", "put", "patch", "delete", "head", "options", "trace"] + REQUEST_PARAM_FIELD_MAP = { + "GET" => {["GET"], "query"}, + "POST" => {["POST"], "form"}, + "COOKIES" => {nil, "header"}, + "META" => {nil, "header"}, + "data" => {["POST", "PUT", "PATCH"], "form"}, + } + REQUEST_PARAM_TYPE_MAP = { + "query" => ["GET"], + "form" => ["GET", "POST", "PUT", "PATCH"], + "cookie" => nil, + "header" => nil, + } def analyze result = [] of Endpoint @@ -13,6 +28,7 @@ class AnalyzerDjango < Analyzer # Django urls root_django_urls_list = search_root_django_urls_list() root_django_urls_list.each do |root_django_urls| + @django_base_path = root_django_urls.basepath get_endpoints(root_django_urls).each do |endpoint| result << endpoint end @@ -30,7 +46,9 @@ class AnalyzerDjango < Analyzer def search_root_django_urls_list : Array(DjangoUrls) root_django_urls_list = [] of DjangoUrls - Dir.glob("#{base_path}/**/*") do |file| + + search_dir = @base_path + Dir.glob("#{search_dir}/**/*") do |file| spawn do begin next if File.directory?(file) @@ -38,9 +56,11 @@ class AnalyzerDjango < Analyzer content = File.read(file, encoding: "utf-8", invalid: :skip) content.scan(REGEX_ROOT_URLCONF) do |match| next if match.size != 2 - filepath = "#{base_path}/#{match[1].gsub(".", "/")}.py" - if File.exists? filepath - root_django_urls_list << DjangoUrls.new("", filepath) + dotted_as_urlconf = match[1].split(".") + relative_path = "#{dotted_as_urlconf.join("/")}.py" + Dir.glob("#{search_dir}/**/#{relative_path}") do |filepath| + basepath = filepath.split("/")[..-(dotted_as_urlconf.size + 1)].join("/") + root_django_urls_list << DjangoUrls.new("", filepath, basepath) end end end @@ -54,64 +74,478 @@ class AnalyzerDjango < Analyzer root_django_urls_list.uniq end + module PackageType + FILE = 0 + CODE = 1 + end + + def travel_package(package_path, dotted_as_names) + travel_package_map = Array(Tuple(String, String, Int32)).new + + py_path = "" + is_travel_positive = false + dotted_as_names_split = dotted_as_names.split(".") + dotted_as_names_split[0..-1].each_with_index do |names, index| + travel_package_path = File.join(package_path, names) + + py_guess = "#{travel_package_path}.py" + if File.directory? travel_package_path + package_path = travel_package_path + is_travel_positive = true + elsif dotted_as_names_split.size - 2 <= index && File.exists? py_guess + py_path = py_guess + is_travel_positive = true + else + break + end + end + + if is_travel_positive == false + return travel_package_map + end + + names = dotted_as_names_split[-1] + names.split(",").each do |name| + _import = name.strip + next if _import == "" + + _alias = nil + if _import.includes? " as " + _import, _alias = _import.split(" as ") + end + + package_type = PackageType::CODE + py_guess = File.join(package_path, "#{_import}.py") + if File.exists? py_guess + package_type = PackageType::FILE + py_path = py_guess + end + + next if py_path == "" + if !_alias.nil? + travel_package_map << {_alias, py_path, package_type} + else + travel_package_map << {_import, py_path, package_type} + end + end + + travel_package_map + end + + def parse_import_packages(url_base_path : String, content : String) + # https://docs.python.org/3/reference/import.html + package_map = {} of String => Tuple(String, Int32) + + offset = 0 + content.each_line do |line| + package_path = @django_base_path + + _from = "" + _imports = "" + _aliases = "" + if line.starts_with? "from" + line.scan(/from\s*([^'"\s\\]*)\s*import\s*(.*)/) do |match| + next if match.size != 3 + _from = match[1] + _imports = match[2] + end + elsif line.starts_with? "import" + line.scan(/import\s*([^'"\s\\]*)/) do |match| + next if match.size != 2 + _imports = match[1] + end + end + + unless _imports == "" + round_bracket_index = line.index('(') + if !round_bracket_index.nil? + # Parse 'import (\n a,\n b,\n c)' pattern + index = offset + round_bracket_index + 1 + while index < content.size && content[index] != ')' + index += 1 + end + _imports = content[(offset + round_bracket_index + 1)..(index - 1)].strip + end + + # Relative path + if _from.starts_with? ".." + package_path = File.join(url_base_path, "..") + _from = _from[2..] + elsif _from.starts_with? "." + package_path = url_base_path + _from = _from[1..] + end + + _imports.split(",").each do |_import| + if _import.starts_with? ".." + package_path = File.join(url_base_path, "..") + elsif _import.starts_with? "." + package_path = url_base_path + end + + dotted_as_names = _import + if _from != "" + dotted_as_names = _from + "." + _import + end + + # Create package map (Hash[name => filepath, ...]) + travel_package_map = travel_package(package_path, dotted_as_names) + next if travel_package_map.size == 0 + travel_package_map.each do |travel_package| + name, filepath, package_type = travel_package + package_map[name] = {filepath, package_type} + end + end + end + + offset += line.size + 1 + end + + package_map + end + def get_endpoints(django_urls : DjangoUrls) : Array(Endpoint) endpoints = [] of Endpoint - paths = get_paths(django_urls) - paths.each do |path| - path = path.gsub("//", "/") - unless path.starts_with?("/") - path = "/#{path}" + url_base_path = File.dirname(django_urls.filepath) + + file = File.open(django_urls.filepath, encoding: "utf-8", invalid: :skip) + content = file.gets_to_end + package_map = parse_import_packages(url_base_path, content) + + # [Temporary Fix] Parse only the string after "urlpatterns = [" + keywords = ["urlpatterns", "=", "["] + keywords.each do |keyword| + if !content.includes? keyword + return endpoints + end + + content = content.split(keyword, 2)[1] + end + + # [TODO] Parse correct urlpatterns from variable concatenation case" + content.scan(REGEX_ROUTE_MAPPING) do |route_match| + next if route_match.size != 3 + route = route_match[1] + route = route.gsub(/^\^/, "").gsub(/\$$/, "") + view = route_match[2].split(",")[0] + url = "/#{@url}/#{django_urls.prefix}/#{route}".gsub(/\/+/, "/") + + new_django_urls = nil + view.scan(REGEX_INCLUDE_URLS) do |include_pattern_match| + # Detect new url configs + next if include_pattern_match.size != 2 + new_route_path = "#{@django_base_path}/#{include_pattern_match[1].gsub(".", "/")}.py" + + if File.exists?(new_route_path) + new_django_urls = DjangoUrls.new("#{django_urls.prefix}#{route}", new_route_path, django_urls.basepath) + get_endpoints(new_django_urls).each do |endpoint| + endpoints << endpoint + end + end end + next if new_django_urls != nil - endpoints << Endpoint.new("#{@url}#{path}", "GET") + if view == "" + endpoints << Endpoint.new(url, "GET") + else + dotted_as_names_split = view.split(".") + + filepath = "" + function_or_class_name = "" + dotted_as_names_split.each_with_index do |name, index| + if (package_map.has_key? name) && (index < dotted_as_names_split.size) + filepath, package_type = package_map[name] + function_or_class_name = name + if package_type == PackageType::FILE && index + 1 < dotted_as_names_split.size + function_or_class_name = dotted_as_names_split[index + 1] + end + + break + end + end + + if filepath != "" + get_endpoint_from_files(url, filepath, function_or_class_name).each do |endpoint| + endpoints << endpoint + end + else + # By default, Django allows requests with methods other than GET as well + # Prevent this flow, we need to improve trace code of 'get_endpoint_from_files() + endpoints << Endpoint.new(url, "GET") + end + end end endpoints end - def get_paths(django_urls : DjangoUrls) - paths = [] of String - content = File.read(django_urls.filepath, encoding: "utf-8", invalid: :skip) - content.scan(REGEX_URL_PATTERNS) do |match| - next if match.size != 2 - paths = mapping_to_path(match[1], django_urls.prefix) + def get_endpoint_from_files(url : String, filepath : String, function_or_class_name : String) + endpoints = Array(Endpoint).new + suspicious_http_methods = ["GET"] + suspicious_params = Array(Param).new + + content = File.read(filepath, encoding: "utf-8", invalid: :skip) + content_lines = content.split "\n" + + # Function Based View + function_start_index = content.index /def\s+#{function_or_class_name}\s*\(/ + if !function_start_index.nil? + function_codeblock = parse_function_or_class(content[function_start_index..]) + if !function_codeblock.nil? + lines = function_codeblock.split "\n" + function_define_line = lines[0] + lines = lines[1..] + + # Verify if the decorator line contains an HTTP method, for instance: + # '@api_view(['POST'])', '@require_POST', '@require_http_methods(["GET", "POST"])' + index = content_lines.index(function_define_line) + if !index.nil? + while index > 0 + index -= 1 + + preceding_definition = content_lines[index] + if preceding_definition.size > 0 && preceding_definition[0] == '@' + HTTP_METHOD_NAMES.each do |http_method_name| + method_name_match = preceding_definition.downcase.match /[^a-zA-Z0-9](#{http_method_name})[^a-zA-Z0-9]/ + if !method_name_match.nil? + suspicious_http_methods << http_method_name.upcase + end + end + end + + break + end + end + + lines.each do |line| + # Check if line has 'request.method == "GET"' similar pattern + if line.includes? "request.method" + suspicious_code = line.split("request.method")[1].strip + HTTP_METHOD_NAMES.each do |http_method_name| + method_name_match = suspicious_code.downcase.match /['"](#{http_method_name})['"]/ + if !method_name_match.nil? + suspicious_http_methods << http_method_name.upcase + end + end + end + + parse_params(line, suspicious_http_methods).each do |param| + suspicious_params << param + end + end + + suspicious_http_methods.uniq.each do |http_method_name| + endpoints << Endpoint.new(url, http_method_name, get_filtered_params(http_method_name, suspicious_params)) + end + + return endpoints + end + end + + # Class Based View + regex_http_method_names = HTTP_METHOD_NAMES.join "|" + class_start_index = content.index /class\s+#{function_or_class_name}\s*[\(:]/ + if !class_start_index.nil? + class_codeblock = parse_function_or_class(content[class_start_index..]) + if !class_codeblock.nil? + lines = class_codeblock.split "\n" + class_define_line = lines[0] + lines = lines[1..] + + # [TODO] Create a graph and use Django internal views + # Suspicious implicit class name for this class + # https://github.com/django/django/blob/main/django/views/generic/edit.py + if class_define_line.includes? "Form" + suspicious_http_methods << "GET" + suspicious_http_methods << "POST" + elsif class_define_line.includes? "Delete" + suspicious_http_methods << "DELETE" + suspicious_http_methods << "POST" + elsif class_define_line.includes? "Create" + suspicious_http_methods << "POST" + elsif class_define_line.includes? "Update" + suspicious_http_methods << "POST" + end + + # Check http methods (django.views.View) + lines.each do |line| + method_function_match = line.match(/\s+def\s+(#{regex_http_method_names})\s*\(/) + if !method_function_match.nil? + suspicious_http_methods << method_function_match[1].upcase + end + + parse_params(line, suspicious_http_methods).each do |param| + suspicious_params << param + end + end + + suspicious_http_methods.uniq.each do |http_method_name| + endpoints << Endpoint.new(url, http_method_name, get_filtered_params(http_method_name, suspicious_params)) + end + + return endpoints + end + end + + # GET is default http method + [Endpoint.new(url, "GET")] + end + + def parse_function_or_class(content : String) + lines = content.split("\n") + + indent_size = 0 + if lines.size > 0 + while indent_size < lines[0].size && lines[0][indent_size] == ' ' + # Only spaces, no tabs + indent_size += 1 + end + + indent_size += INDENT_SPACE_SIZE end - paths + if indent_size > 0 + double_quote_open, single_quote_open = [false] * 2 + double_comment_open, single_comment_open = [false] * 2 + end_index = lines[0].size + 1 + lines[1..].each do |line| + line_index = 0 + clear_line = line + while line_index < line.size + if line_index < line.size - 2 + if !single_quote_open && !double_quote_open + if !double_comment_open && line[line_index..line_index + 2] == "'''" + single_comment_open = !single_comment_open + line_index += 3 + next + elsif !single_comment_open && line[line_index..line_index + 2] == "\"\"\"" + double_comment_open = !double_comment_open + line_index += 3 + next + end + end + end + + if !single_comment_open && !double_comment_open + if !single_quote_open && line[line_index] == '"' && line[line_index - 1] != '\\' + double_quote_open = !double_quote_open + elsif !double_quote_open && line[line_index] == '\'' && line[line_index - 1] != '\\' + single_quote_open = !single_quote_open + elsif !single_quote_open && !double_quote_open && line[line_index] == '#' && line[line_index - 1] != '\\' + clear_line = line[..(line_index - 1)] + break + end + end + + # [TODO] Remove comments on codeblock + line_index += 1 + end + + open_status = single_comment_open || double_comment_open || single_quote_open || double_quote_open + if clear_line[0..(indent_size - 1)].strip == "" || open_status + end_index += line.size + 1 + else + break + end + end + + end_index -= 1 + return content[..end_index].strip + end + + nil end - def mapping_to_path(content : String, prefix : String = "") - paths = Array(String).new - content.scan(REGEX_URL_MAPPING) do |match| - next if match.size != 3 - path = match[1] - view = match[2] + def parse_params(line : String, endpoint_methods : Array(String)) + suspicious_params = Array(Param).new + + if line.includes? "request." + REQUEST_PARAM_FIELD_MAP.each do |field_name, tuple| + field_methods, noir_param_type = tuple + matches = line.scan(/request\.#{field_name}\[['"]([^'"]*)['"]\]/) + if matches.size == 0 + matches = line.scan(/request\.#{field_name}\.get\(['"]([^'"]*)['"]/) + end - path = path.gsub(/ /, "") - path = path.gsub(/^\^/, "") - path = path.gsub(/\$$/, "") + if matches.size != 0 + matches.each do |match| + next if match.size != 2 + param_name = match[1] + if field_name == "META" + if param_name.starts_with? "HTTP_" + param_name = param_name[5..] + end + elsif noir_param_type == "header" + if field_name == "COOKIES" + param_name = "Cookie['#{param_name}']" + end + end - filepath = nil - view.scan(REGEX_INCLUDE_URLS) do |include_pattern_match| - next if include_pattern_match.size != 2 - filepath = "#{base_path}/#{include_pattern_match[1].gsub(".", "/")}.py" + # If it receives a specific parameter, it is considered to allow the method. + if !field_methods.nil? + field_methods.each do |field_method| + if !endpoint_methods.includes? field_method + endpoint_methods << field_method + end + end + end + + suspicious_params << Param.new(param_name, "", noir_param_type) + end + end + end + end + + if line.includes? "form.cleaned_data" + matches = line.scan(/form\.cleaned_data\[['"]([^'"]*)['"]\]/) + if matches.size == 0 + matches = line.scan(/form\.cleaned_data\.get\(['"]([^'"]*)['"]/) + end + + if matches.size != 0 + matches.each do |match| + next if match.size != 2 + suspicious_params << Param.new(match[1], "", "form") + end + end + end + + suspicious_params + end - if File.exists?(filepath) - new_django_urls = DjangoUrls.new("#{prefix}#{path}", filepath) - new_paths = get_paths(new_django_urls) - new_paths.each do |new_path| - paths << new_path + def get_filtered_params(method : String, params : Array(Param)) + filtered_params = Array(Param).new + upper_method = method.upcase + + params.each do |param| + is_support_param = false + support_methods = REQUEST_PARAM_TYPE_MAP.fetch(param.param_type, nil) + if !support_methods.nil? + support_methods.each do |support_method| + if upper_method == support_method.upcase + is_support_param = true + elsif support_method.upcase == "GET" && param.param_type == "query" + # The GET method allows parameters to be used in other methods as well. + is_support_param = true end end + else + is_support_param = true end - unless path.starts_with?("/") - path = "/#{path}" + filtered_params.each do |filtered_param| + if filtered_param.name == param.name && filtered_param.param_type == param.param_type + is_support_param = false + break + end + end + + if is_support_param + filtered_params << param end - paths << "#{prefix}#{path}" end - paths + filtered_params end end @@ -122,9 +556,12 @@ end struct DjangoUrls include JSON::Serializable - property prefix, filepath + property prefix, filepath, basepath - def initialize(@prefix : String, @filepath : String) + def initialize(@prefix : String, @filepath : String, @basepath : String) + if !File.directory? @basepath + raise "The basepath for DjangoUrls (#{@basepath}) does not exist or is not a directory." + end end end @@ -133,5 +570,8 @@ struct DjangoView property prefix, filepath, name def initialize(@prefix : String, @filepath : String, @name : String) + if !File.directory? @filepath + raise "The filepath for DjangoView (#{@filepath}) does not exist." + end end end From 00670c8f5b1fe0a3a3348da0a0ea6025906502a7 Mon Sep 17 00:00:00 2001 From: ksg97031 Date: Sun, 10 Sep 2023 15:54:18 +0900 Subject: [PATCH 5/7] Update strings regex to handle 'r' and 'f' prefix --- src/analyzer/analyzers/analyzer_django.cr | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/analyzer/analyzers/analyzer_django.cr b/src/analyzer/analyzers/analyzer_django.cr index ce8144ea..898102bb 100644 --- a/src/analyzer/analyzers/analyzer_django.cr +++ b/src/analyzer/analyzers/analyzer_django.cr @@ -462,9 +462,9 @@ class AnalyzerDjango < Analyzer if line.includes? "request." REQUEST_PARAM_FIELD_MAP.each do |field_name, tuple| field_methods, noir_param_type = tuple - matches = line.scan(/request\.#{field_name}\[['"]([^'"]*)['"]\]/) + matches = line.scan(/request\.#{field_name}\[[rf]?['"]([^'"]*)['"]\]/) if matches.size == 0 - matches = line.scan(/request\.#{field_name}\.get\(['"]([^'"]*)['"]/) + matches = line.scan(/request\.#{field_name}\.get\([rf]?['"]([^'"]*)['"]/) end if matches.size != 0 @@ -497,9 +497,9 @@ class AnalyzerDjango < Analyzer end if line.includes? "form.cleaned_data" - matches = line.scan(/form\.cleaned_data\[['"]([^'"]*)['"]\]/) + matches = line.scan(/form\.cleaned_data\[[rf]?['"]([^'"]*)['"]\]/) if matches.size == 0 - matches = line.scan(/form\.cleaned_data\.get\(['"]([^'"]*)['"]/) + matches = line.scan(/form\.cleaned_data\.get\([rf]?['"]([^'"]*)['"]/) end if matches.size != 0 From e77f188ae6e075422057034c2d0cfa47ef2b2b27 Mon Sep 17 00:00:00 2001 From: ksg97031 Date: Sun, 10 Sep 2023 15:59:11 +0900 Subject: [PATCH 6/7] Code refactoring --- src/analyzer/analyzers/analyzer_django.cr | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/analyzer/analyzers/analyzer_django.cr b/src/analyzer/analyzers/analyzer_django.cr index 898102bb..7d52b14d 100644 --- a/src/analyzer/analyzers/analyzer_django.cr +++ b/src/analyzer/analyzers/analyzer_django.cr @@ -16,7 +16,7 @@ class AnalyzerDjango < Analyzer "data" => {["POST", "PUT", "PATCH"], "form"}, } REQUEST_PARAM_TYPE_MAP = { - "query" => ["GET"], + "query" => nil, "form" => ["GET", "POST", "PUT", "PATCH"], "cookie" => nil, "header" => nil, @@ -524,9 +524,6 @@ class AnalyzerDjango < Analyzer support_methods.each do |support_method| if upper_method == support_method.upcase is_support_param = true - elsif support_method.upcase == "GET" && param.param_type == "query" - # The GET method allows parameters to be used in other methods as well. - is_support_param = true end end else From 48b4785f9ea32bf0cd37c40bca64aff60d73d11c Mon Sep 17 00:00:00 2001 From: ksg97031 Date: Sun, 10 Sep 2023 16:21:20 +0900 Subject: [PATCH 7/7] Improve Flask Analyzer (method, param, header) --- README.md | 2 +- .../fixtures/flask/__init__.py | 0 spec/functional_test/fixtures/flask/app.py | 91 +++++ .../fixtures/flask/requirements.txt | 3 + .../fixtures/flask/static/css/example.css | 2 + .../fixtures/flask/static/js/example.js | 1 + .../fixtures/flask/templates/__init__.py | 0 .../fixtures/flask/templates/error.html | 8 + .../fixtures/flask/templates/index.html | 13 + .../fixtures/flask/templates/login.html | 13 + .../fixtures/flask/templates/sign.html | 13 + spec/functional_test/fixtures/flask/utils.py | 6 + .../testers/python_flask_spec.cr | 16 + src/analyzer/analyzers/analyzer_flask.cr | 317 +++++++++++++++++- 14 files changed, 473 insertions(+), 12 deletions(-) create mode 100644 spec/functional_test/fixtures/flask/__init__.py create mode 100644 spec/functional_test/fixtures/flask/app.py create mode 100644 spec/functional_test/fixtures/flask/requirements.txt create mode 100644 spec/functional_test/fixtures/flask/static/css/example.css create mode 100644 spec/functional_test/fixtures/flask/static/js/example.js create mode 100644 spec/functional_test/fixtures/flask/templates/__init__.py create mode 100644 spec/functional_test/fixtures/flask/templates/error.html create mode 100644 spec/functional_test/fixtures/flask/templates/index.html create mode 100644 spec/functional_test/fixtures/flask/templates/login.html create mode 100644 spec/functional_test/fixtures/flask/templates/sign.html create mode 100644 spec/functional_test/fixtures/flask/utils.py create mode 100644 spec/functional_test/testers/python_flask_spec.cr diff --git a/README.md b/README.md index 00acf97d..e8b730a2 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ | Go | Echo | ✅ | ✅ | ✅ | ✅ | X | | Go | Gin | ✅ | ✅ | ✅ | ✅ | X | | Python | Django | ✅ | ✅ | ✅ | ✅ | X | -| Python | Flask | ✅ | X | X | X | X | +| Python | Flask | ✅ | ✅ | ✅ | ✅ | X | | Ruby | Rails | ✅ | ✅ | ✅ | ✅ | X | | Ruby | Sinatra | ✅ | ✅ | ✅ | ✅ | X | | Php | | ✅ | ✅ | ✅ | ✅ | X | diff --git a/spec/functional_test/fixtures/flask/__init__.py b/spec/functional_test/fixtures/flask/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/spec/functional_test/fixtures/flask/app.py b/spec/functional_test/fixtures/flask/app.py new file mode 100644 index 00000000..b1735e92 --- /dev/null +++ b/spec/functional_test/fixtures/flask/app.py @@ -0,0 +1,91 @@ +import sys +import json +import hashlib +from flask import Flask, render_template, request, session, jsonify +from database import db_session +from models import User +from utils import get_hash + +app = Flask(__name__) +app.secret_key = "dd2e7b987b357908fac0118ecdf0d3d2cae7b5a635f802d6" # random generate + +@app.teardown_appcontext +def shutdown_session(exception=None): + db_session.remove() + +@app.route('/sign', methods=['GET', 'POST']) +def sign_sample(): + if request.method == 'POST': + username = request.form['username'] + password = get_hash(request.form['password'], app.secret_key) + if User.query.filter(User.name == username).first(): + return render_template('error.html') + + u = User(username, password) + db_session.add(u) + db_session.commit() + return render_template('login.html') + + return render_template('sign.html') + +@app.route('/login', methods=['POST']) +def login_sample(): + if request.method == 'POST': + username = request.form['username'] + password = get_hash(request.form['password'], app.secret_key) + if User.query.filter(User.name == username and User.password == password).first(): + session['logged_in'] = True + session['username'] = username + return render_template('index.html') + else: + return "Fail" + + return render_template('login.html') + +@app.route('/create_record', methods=['PUT']) +def create_record(): + record = json.loads(request.data) + with open('/tmp/data.txt', 'r') as f: + data = f.read() + if not data: + records = [record] + else: + records = json.loads(data) + records.append(record) + with open('/tmp/data.txt', 'w') as f: + f.write(json.dumps(records, indent=2)) + return jsonify(record) + +@app.route('/delete_record', methods=['DELETE']) +def delte_record(): + record = json.loads(request.data) + new_records = [] + with open('/tmp/data.txt', 'r') as f: + data = f.read() + records = json.loads(data) + for r in records: + if r['name'] == record['name']: + continue + new_records.append(r) + with open('/tmp/data.txt', 'w') as f: + f.write(json.dumps(new_records, indent=2)) + return jsonify(record) + +@app.route('/get_ip', methods=['GET']) +def json_sample(): + data = {'ip': request.headers.get('X-Forwarded-For', request.remote_addr)} + + return jsonify(data), 200 + +@app.route('/') +def index(): + return render_template('index.html') + + +if __name__ == "__main__": + port = 80 + if len(sys.argv) > 1: + port = int(sys.argv[1]) + + app.run(host='0.0.0.0', port=port) + diff --git a/spec/functional_test/fixtures/flask/requirements.txt b/spec/functional_test/fixtures/flask/requirements.txt new file mode 100644 index 00000000..289427c4 --- /dev/null +++ b/spec/functional_test/fixtures/flask/requirements.txt @@ -0,0 +1,3 @@ +flask +flask-sqlalchemy +sqlalchemy diff --git a/spec/functional_test/fixtures/flask/static/css/example.css b/spec/functional_test/fixtures/flask/static/css/example.css new file mode 100644 index 00000000..ff896d78 --- /dev/null +++ b/spec/functional_test/fixtures/flask/static/css/example.css @@ -0,0 +1,2 @@ +p { +} diff --git a/spec/functional_test/fixtures/flask/static/js/example.js b/spec/functional_test/fixtures/flask/static/js/example.js new file mode 100644 index 00000000..18883c03 --- /dev/null +++ b/spec/functional_test/fixtures/flask/static/js/example.js @@ -0,0 +1 @@ +console.log('example js') diff --git a/spec/functional_test/fixtures/flask/templates/__init__.py b/spec/functional_test/fixtures/flask/templates/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/spec/functional_test/fixtures/flask/templates/error.html b/spec/functional_test/fixtures/flask/templates/error.html new file mode 100644 index 00000000..102771e0 --- /dev/null +++ b/spec/functional_test/fixtures/flask/templates/error.html @@ -0,0 +1,8 @@ + + +test + + + Error Page + + diff --git a/spec/functional_test/fixtures/flask/templates/index.html b/spec/functional_test/fixtures/flask/templates/index.html new file mode 100644 index 00000000..e0a3093a --- /dev/null +++ b/spec/functional_test/fixtures/flask/templates/index.html @@ -0,0 +1,13 @@ + + + test + + + + +

test

+ {% if session['logged_in'] %} +

{{ session['username'] }}

+ {% endif %} + + diff --git a/spec/functional_test/fixtures/flask/templates/login.html b/spec/functional_test/fixtures/flask/templates/login.html new file mode 100644 index 00000000..55299224 --- /dev/null +++ b/spec/functional_test/fixtures/flask/templates/login.html @@ -0,0 +1,13 @@ + + +test + + +

Login Page

+
+ ID + PW
+ +
+ + diff --git a/spec/functional_test/fixtures/flask/templates/sign.html b/spec/functional_test/fixtures/flask/templates/sign.html new file mode 100644 index 00000000..efb0fb01 --- /dev/null +++ b/spec/functional_test/fixtures/flask/templates/sign.html @@ -0,0 +1,13 @@ + + +test + + +

Account Regist Page

+
+ ID + PW
+ +
+ + diff --git a/spec/functional_test/fixtures/flask/utils.py b/spec/functional_test/fixtures/flask/utils.py new file mode 100644 index 00000000..17145075 --- /dev/null +++ b/spec/functional_test/fixtures/flask/utils.py @@ -0,0 +1,6 @@ +import hashlib + +def get_hash(data, salt): + m = len(salt)//2 + sdata = salt[:m] + data + salt[m:] + return hashlib.sha256(sdata.encode('utf-8')).hexdigest() diff --git a/spec/functional_test/testers/python_flask_spec.cr b/spec/functional_test/testers/python_flask_spec.cr new file mode 100644 index 00000000..91b205ec --- /dev/null +++ b/spec/functional_test/testers/python_flask_spec.cr @@ -0,0 +1,16 @@ +require "../func_spec.cr" + +extected_endpoints = [ + Endpoint.new("/sign", "GET"), + Endpoint.new("/sign", "POST", [Param.new("username", "", "form"), Param.new("password", "", "form")]), + Endpoint.new("/login", "POST", [Param.new("username", "", "form"), Param.new("password", "", "form")]), + Endpoint.new("/create_record", "PUT"), + Endpoint.new("/delete_record", "DELETE", [Param.new("name", "", "json")]), + Endpoint.new("/get_ip", "GET", [Param.new("X-Forwarded-For", "", "header")]), + Endpoint.new("/", "GET"), +] + +FunctionalTester.new("fixtures/flask/", { + :techs => 1, + :endpoints => 7, +}, extected_endpoints).test_all diff --git a/src/analyzer/analyzers/analyzer_flask.cr b/src/analyzer/analyzers/analyzer_flask.cr index 04c1c01c..810df6aa 100644 --- a/src/analyzer/analyzers/analyzer_flask.cr +++ b/src/analyzer/analyzers/analyzer_flask.cr @@ -1,19 +1,85 @@ require "../../models/analyzer" class AnalyzerFlask < Analyzer + REGEX_PYTHON_VARIABLE_NAME = "[a-zA-Z_][a-zA-Z0-9_]*" + HTTP_METHOD_NAMES = ["get", "post", "put", "patch", "delete", "head", "options", "trace"] + INDENT_SPACE_SIZE = 4 + + # https://stackoverflow.com/a/16664376 + # https://tedboy.github.io/flask/generated/generated/flask.Request.html + REQUEST_PARAM_FIELD_MAP = { + "data" => {["POST", "PUT", "PATCH", "DELETE"], "form"}, + "args" => {["GET"], "query"}, + "form" => {["POST", "PUT", "PATCH", "DELETE"], "form"}, + "files" => {["POST", "PUT", "PATCH", "DELETE"], "form"}, + "values" => {["GET", "POST", "PUT", "PATCH", "DELETE"], "query"}, + "json" => {["POST", "PUT", "PATCH", "DELETE"], "json"}, + "cookie" => {nil, "header"}, + "headers" => {nil, "header"}, + } + + REQUEST_PARAM_TYPE_MAP = { + "query" => nil, + "form" => ["POST", "PUT", "PATCH", "DELETE"], + "json" => ["POST", "PUT", "PATCH", "DELETE"], + "cookie" => nil, + "header" => nil, + } + def analyze - # Source Analysis - Dir.glob("#{base_path}/**/*") do |path| + blueprint_prefix_map = {} of String => String + + Dir.glob("#{base_path}/**/*.py") do |path| next if File.directory?(path) - if File.exists?(path) && File.extname(path) == ".py" - File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file| - file.each_line do |line| - line.strip.scan(/@app\.route\((.*)\)/) do |match| - if match.size > 0 - splited = match[0].split("(") - if splited.size > 1 - endpoint_path = splited[1].gsub("\"", "").gsub("'", "").gsub(")", "").gsub(" ", "") - result << Endpoint.new("#{url}#{endpoint_path}", "GET") + File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file| + file.each_line do |line| + # [TODO] We should be cautious about instance replace with other variable + match = line.match /(#{REGEX_PYTHON_VARIABLE_NAME})\s*=\s*Flask\s*\(/ + if !match.nil? + flask_instance_name = match[1] + if !blueprint_prefix_map.has_key? flask_instance_name + blueprint_prefix_map[flask_instance_name] = "" + end + end + + # Common flask instance name + if !blueprint_prefix_map.has_key? "app" + blueprint_prefix_map["app"] = "" + end + + # https://flask.palletsprojects.com/en/2.3.x/tutorial/views/ + match = line.match /(#{REGEX_PYTHON_VARIABLE_NAME})\s*=\s*Blueprint\s*\(/ + if !match.nil? + prefix = "" + blueprint_instance_name = match[1] + param_codes = line.split("Blueprint", 2)[1] + prefix_match = param_codes.match /url_prefix\s=\s['"](['"]*)['"]/ + if !prefix_match.nil? && prefix_match.size == 2 + prefix = prefix_match[1] + end + + if !blueprint_prefix_map.has_key? blueprint_instance_name + blueprint_prefix_map[blueprint_instance_name] = prefix + end + end + + # [TODO] We're not concerned with nesting blueprints at the moment + # https://flask.palletsprojects.com/en/2.3.x/blueprints/#nesting-blueprints + + # [Temporary Addition] register_view + blueprint_prefix_map.each do |blueprint_name, blueprint_prefix| + register_views_match = line.match /#{blueprint_name},\s*routes\s*=\s*(.*)\)/ + if !register_views_match.nil? + route_paths = register_views_match[1] + route_paths.scan /['"]([^'"]*)['"]/ do |path_str_match| + if !path_str_match.nil? && path_str_match.size == 2 + route_path = path_str_match[1] + # [TODO] Parse methods from reference views + route_url = "#{@url}#{blueprint_prefix}#{route_path}" + if !route_url.starts_with? "/" + route_url = "/#{route_url}" + end + result << Endpoint.new(route_url, "GET") end end end @@ -21,10 +87,239 @@ class AnalyzerFlask < Analyzer end end end + + Dir.glob("#{base_path}/**/*.py") do |path| + next if File.directory?(path) + source = File.read(path, encoding: "utf-8", invalid: :skip) + lines = source.split "\n" + + line_index = 0 + while line_index < lines.size + line = lines[line_index] + blueprint_prefix_map.each do |flask_instance_name, prefix| + # https://flask.palletsprojects.com/en/2.3.x/quickstart/#http-methods + line.scan(/@#{flask_instance_name}\.route\([rf]?['"]([^'"]*)['"](.*)/) do |match| + if match.size > 0 + route_path = match[1] + extra_params = match[2] + + # Pass decorator lines + codeline_index = line_index + while codeline_index < lines.size + decorator_match = lines[codeline_index].match /\s*@/ + if !decorator_match.nil? + codeline_index += 1 + next + end + + break + end + + codeblock = parse_function_or_class(lines[codeline_index..].join("\n")) + next if codeblock.nil? + codeblock_lines = codeblock.split("\n") + codeblock_lines = codeblock_lines[1..] + + get_endpoints(route_path, extra_params, codeblock_lines, prefix).each do |endpoint| + result << endpoint + end + end + end + end + line_index += 1 + end + end Fiber.yield result end + + def get_endpoints(route_path : String, extra_params : String, codeblock_lines : Array(String), prefix : String) + endpoints = [] of Endpoint + suspicious_http_methods = [] of String + suspicious_params = [] of Param + + if (!prefix.ends_with? "/") && (!route_path.starts_with? "/") + prefix = "#{prefix}/" + end + + # Parse declared methods + methods_match = extra_params.match /methods\s*=\s*(.*)/ + if !methods_match.nil? && methods_match.size == 2 + declare_methods = methods_match[1].downcase + HTTP_METHOD_NAMES.each do |method_name| + if declare_methods.includes? method_name + suspicious_http_methods << method_name.upcase + end + end + else + suspicious_http_methods << "GET" + end + + json_variable_names = [] of String + # Parse JSON variable names + codeblock_lines.each do |codeblock_line| + match = codeblock_line.match /([a-zA-Z_][a-zA-Z0-9_]*).*=\s*json\.loads\(request\.data/ + if !match.nil? && match.size == 2 && !json_variable_names.includes?(match[1]) + json_variable_names << match[1] + end + + match = codeblock_line.match /([a-zA-Z_][a-zA-Z0-9_]*).*=\s*request\.json/ + if !match.nil? && match.size == 2 && !json_variable_names.includes?(match[1]) + json_variable_names << match[1] + end + end + + # Parse declared parameters + codeblock_lines.each do |codeblock_line| + REQUEST_PARAM_FIELD_MAP.each do |field_name, tuple| + _, noir_param_type = tuple + matches = codeblock_line.scan(/request\.#{field_name}\[[rf]?['"]([^'"]*)['"]\]/) + if matches.size == 0 + matches = codeblock_line.scan(/request\.#{field_name}\.get\([rf]?['"]([^'"]*)['"]/) + end + if matches.size == 0 + noir_param_type = "json" + json_variable_names.each do |json_variable_name| + matches = codeblock_line.scan(/[^a-zA-Z_]#{json_variable_name}\[[rf]?['"]([^'"]*)['"]\]/) + if matches.size == 0 + matches = codeblock_line.scan(/[^a-zA-Z_]#{json_variable_name}\.get\([rf]?['"]([^'"]*)['"]/) + end + + if matches.size > 0 + break + end + end + end + + matches.each do |parameter_match| + next if parameter_match.size != 2 + param_name = parameter_match[1] + if noir_param_type == "header" + if field_name == "cookie" + param_name = "Cookie['#{param_name}']" + end + end + + suspicious_params << Param.new(param_name, "", noir_param_type) + end + end + end + + suspicious_http_methods.uniq.each do |http_method_name| + if (!prefix.ends_with? "/") && (!route_path.starts_with? "/") + prefix = "#{prefix}/" + end + + route_url = "#{@url}#{prefix}#{route_path}" + if !route_url.starts_with? "/" + route_url = "/#{route_url}" + end + + params = get_filtered_params(http_method_name, suspicious_params) + endpoints << Endpoint.new(route_url, http_method_name, params) + end + + endpoints + end + + def parse_function_or_class(content : String) + # [TODO] Split to other module (duplicated method with analyzer_django) + indent_size = 0 + lines = content.split "\n" + if lines.size > 0 + while indent_size < lines[0].size && lines[0][indent_size] == ' ' + # Only spaces, no tabs + indent_size += 1 + end + + indent_size += INDENT_SPACE_SIZE + end + + if indent_size > 0 + double_quote_open, single_quote_open = [false] * 2 + double_comment_open, single_comment_open = [false] * 2 + end_index = lines[0].size + 1 + lines[1..].each do |line| + line_index = 0 + clear_line = line + while line_index < line.size + if line_index < line.size - 2 + if !single_quote_open && !double_quote_open + if !double_comment_open && line[line_index..line_index + 2] == "'''" + single_comment_open = !single_comment_open + line_index += 3 + next + elsif !single_comment_open && line[line_index..line_index + 2] == "\"\"\"" + double_comment_open = !double_comment_open + line_index += 3 + next + end + end + end + + if !single_comment_open && !double_comment_open + if !single_quote_open && line[line_index] == '"' && line[line_index - 1] != '\\' + double_quote_open = !double_quote_open + elsif !double_quote_open && line[line_index] == '\'' && line[line_index - 1] != '\\' + single_quote_open = !single_quote_open + elsif !single_quote_open && !double_quote_open && line[line_index] == '#' && line[line_index - 1] != '\\' + clear_line = line[..(line_index - 1)] + break + end + end + + # [TODO] Remove comments on codeblock + line_index += 1 + end + + open_status = single_comment_open || double_comment_open || single_quote_open || double_quote_open + if clear_line[0..(indent_size - 1)].strip == "" || open_status + end_index += line.size + 1 + else + break + end + end + + end_index -= 1 + return content[..end_index].strip + end + + nil + end + + def get_filtered_params(method : String, params : Array(Param)) + # [TODO] Split to other module (duplicated method with analyzer_django) + filtered_params = Array(Param).new + upper_method = method.upcase + + params.each do |param| + is_support_param = false + support_methods = REQUEST_PARAM_TYPE_MAP.fetch(param.param_type, nil) + if !support_methods.nil? + support_methods.each do |support_method| + if upper_method == support_method.upcase + is_support_param = true + end + end + else + is_support_param = true + end + + filtered_params.each do |filtered_param| + if filtered_param.name == param.name && filtered_param.param_type == param.param_type + is_support_param = false + break + end + end + + if is_support_param + filtered_params << param + end + end + + filtered_params + end end def analyzer_flask(options : Hash(Symbol, String))