Skip to content

Commit

Permalink
Merge branch 'main' into AzCC_CleanupRestProxyCode
Browse files Browse the repository at this point in the history
  • Loading branch information
alzimmermsft authored Dec 20, 2024
2 parents c9f403c + c793ab0 commit d2f8f7f
Show file tree
Hide file tree
Showing 517 changed files with 4,140 additions and 27,251 deletions.
5 changes: 5 additions & 0 deletions eng/pipelines/templates/jobs/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -278,6 +278,11 @@ jobs:
filePath: $(Build.SourcesDirectory)/eng/versioning/pom_file_version_scanner.ps1
arguments: -Debug:$$(IsDebug)

- task: PythonScript@0
displayName: 'Verify version text files'
inputs:
scriptPath: eng/versioning/find_unused_dependencies.py

- template: /eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml
parameters:
JobType: 'Analyze'
Expand Down
5 changes: 5 additions & 0 deletions eng/pipelines/templates/stages/archetype-sdk-client-patch.yml
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,11 @@ extends:
filePath: eng/versioning/pom_file_version_scanner.ps1
arguments: -Debug:$$(IsDebug)

- task: PythonScript@0
displayName: 'Verify version text files'
inputs:
scriptPath: eng/versioning/find_unused_dependencies.py

# Do all the verifications for all the artifacts after the sparse-checkout.
- pwsh: |
$ArtifactsList = $('$(ArtifactsJson)' | ConvertFrom-Json | Select-Object name, groupId)
Expand Down
87 changes: 0 additions & 87 deletions eng/versioning/external_dependencies.txt
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,9 @@ com.fasterxml.jackson.core:jackson-core;2.17.2
com.fasterxml.jackson.core:jackson-databind;2.17.2
com.fasterxml.jackson.dataformat:jackson-dataformat-avro;2.17.2
com.fasterxml.jackson.dataformat:jackson-dataformat-xml;2.17.2
com.fasterxml.jackson.datatype:jackson-datatype-jdk8;2.17.2
com.fasterxml.jackson.datatype:jackson-datatype-jsr310;2.17.2
com.fasterxml.jackson.module:jackson-module-afterburner;2.17.2
com.fasterxml.jackson.module:jackson-module-blackbird;2.17.2
com.fasterxml.jackson.module:jackson-module-parameter-names;2.17.2
com.fasterxml.jackson.module:jackson-module-scala_2.12;2.17.2
com.github.spotbugs:spotbugs;4.8.3
com.github.spotbugs:spotbugs-annotations;4.8.3
Expand All @@ -30,28 +28,17 @@ com.google.guava:guava;33.1.0-jre
com.h2database:h2;2.2.220
com.h3xstream.findsecbugs:findsecbugs-plugin;1.9.0
com.knuddels:jtokkit;1.0.0
com.microsoft.azure:applicationinsights-core;3.4.1
com.microsoft.azure:azure-annotations;1.7.0
com.microsoft.azure:azure-arm-client-runtime;1.7.14
com.microsoft.azure:azure-client-authentication;1.7.14
com.microsoft.azure:azure-client-runtime;1.7.14
com.microsoft.azure:azure-core;0.9.8
com.microsoft.azure:azure-cosmos;3.7.3
com.microsoft.azure:azure-eventhubs-eph;3.3.0
com.microsoft.azure:azure-keyvault-cryptography;1.2.2
com.microsoft.azure:azure-media;0.9.8
com.microsoft.azure:azure-servicebus;3.6.1
com.microsoft.azure:azure-servicebus-jms;1.0.0
com.microsoft.azure:qpid-proton-j-extensions;1.2.5
com.microsoft.azure.sdk.iot:iot-service-client;1.30.0
com.microsoft.rest:client-runtime;1.7.4
com.microsoft.rest.v2:client-runtime;2.1.1
com.microsoft.sqlserver:mssql-jdbc;10.2.3.jre8
com.microsoft.azure:spring-data-cosmosdb;2.3.0
com.microsoft.azure:azure-functions-maven-plugin;1.30.0
com.microsoft.azure.functions:azure-functions-java-library;2.2.0
com.mysql:mysql-connector-j;9.0.0
com.squareup.okhttp3:mockwebserver;4.12.0
com.squareup.okhttp3:okhttp;4.12.0
commons-codec:commons-codec;1.15
commons-net:commons-net;3.9.0
Expand All @@ -66,22 +53,16 @@ io.netty:netty-common;4.1.115.Final
io.netty:netty-codec;4.1.115.Final
io.netty:netty-codec-http;4.1.115.Final
io.netty:netty-codec-http2;4.1.115.Final
io.netty:netty-transport;4.1.115.Final
io.netty:netty-handler;4.1.115.Final
io.netty:netty-handler-proxy;4.1.115.Final
io.netty:netty-tcnative;2.0.69.Final
io.netty:netty-tcnative-boringssl-static;2.0.69.Final
io.netty:netty-transport-native-epoll;4.1.115.Final
io.netty:netty-transport-native-unix-common;4.1.115.Final
io.netty:netty-transport-native-kqueue;4.1.115.Final
io.projectreactor.netty:reactor-netty;1.0.48
io.projectreactor.netty:reactor-netty-http;1.0.48
io.projectreactor:reactor-core;3.4.41
io.vertx:vertx-codegen;4.5.10
io.vertx:vertx-core;4.5.10
io.vertx:vertx-reactive-streams;4.5.10
javax.annotation:javax.annotation-api;1.3.2
javax.json:javax.json-api;1.1.4
javax.websocket:javax.websocket-api;1.1
org.apache.commons:commons-compress;1.26.0
org.apache.ant:ant;1.10.14
Expand All @@ -95,7 +76,6 @@ org.apache.logging.log4j:log4j-slf4j-impl;2.17.2
org.apache.maven:maven-core;3.9.9
org.apache.maven.archetype:archetype-packaging;3.3.0
org.apache.maven.plugins:maven-archetype-plugin;3.3.0
org.apache.maven.plugins:maven-gpg-plugin;3.2.7
org.apache.qpid:proton-j;0.34.1
org.asynchttpclient:async-http-client;2.12.1
org.codehaus.groovy:groovy-jsr223;3.0.19
Expand All @@ -119,66 +99,12 @@ testdep_net.bytebuddy:byte-buddy;1.15.5
testdep_net.bytebuddy:byte-buddy-agent;1.15.5

## Spring boot dependency versions
org.springframework.boot:spring-boot-dependencies;2.7.18
org.springframework.boot:spring-boot-actuator;2.7.18
org.springframework.boot:spring-boot-actuator-autoconfigure;2.7.18
org.springframework.boot:spring-boot-autoconfigure-processor;2.7.18
org.springframework.boot:spring-boot-autoconfigure;2.7.18
org.springframework.boot:spring-boot-configuration-metadata;2.7.18
org.springframework.boot:spring-boot-configuration-processor;2.7.18
org.springframework.boot:spring-boot-maven-plugin;2.7.18
org.springframework.boot:spring-boot-starter-actuator;2.7.18
org.springframework.boot:spring-boot-starter-aop;2.7.18
org.springframework.boot:spring-boot-starter-cache;2.7.18
org.springframework.boot:spring-boot-starter-data-redis;2.7.18
org.springframework.boot:spring-boot-starter-data-jdbc;2.7.18
org.springframework.boot:spring-boot-starter-integration;2.7.18
org.springframework.boot:spring-boot-starter-logging;2.7.18
org.springframework.boot:spring-boot-starter-test;2.7.18
org.springframework.boot:spring-boot-starter-validation;2.7.18
org.springframework.boot:spring-boot-starter-web;2.7.18
org.springframework.boot:spring-boot-starter;2.7.18
org.springframework.boot:spring-boot-test;2.7.18
org.springframework.boot:spring-boot;2.7.18
org.springframework.data:spring-data-commons;2.7.18
org.springframework.data:spring-data-mongodb;3.4.18
org.springframework.data:spring-data-redis;2.7.18
org.springframework.experimental:spring-aot;0.11.4
org.springframework.experimental:spring-aot-maven-plugin;0.11.4
org.springframework.integration:spring-integration-core;5.5.20
org.springframework.kafka:spring-kafka;2.9.11
org.springframework.security:spring-security-config;5.7.11
org.springframework.security:spring-security-core;5.7.11
org.springframework.security:spring-security-oauth2-client;5.7.11
org.springframework.security:spring-security-oauth2-resource-server;5.7.11
org.springframework.security:spring-security-oauth2-core;5.7.11
org.springframework.security:spring-security-oauth2-jose;5.7.11
org.springframework.security:spring-security-web;5.7.11
org.springframework:spring-beans;5.3.32
org.springframework:spring-context-support;5.3.32
org.springframework:spring-context;5.3.32
org.springframework:spring-core;5.3.32
org.springframework:spring-expression;5.3.32
org.springframework:spring-jdbc;5.3.32
org.springframework:spring-jms;5.3.32
org.springframework:spring-messaging;5.3.32
org.springframework:spring-tx;5.3.32
org.springframework:spring-web;5.3.32
org.springframework:spring-webmvc;5.3.32
org.springframework:spring-test;5.3.32
org.springframework.retry:spring-retry;1.3.4
org.springframework.boot:spring-boot-starter-parent;2.7.18

## Spring cloud dependency versions
org.springframework.cloud:spring-cloud-dependencies;2021.0.9
org.springframework.cloud:spring-cloud-starter-bootstrap;3.1.8
org.springframework.cloud:spring-cloud-bus;3.1.3
org.springframework.cloud:spring-cloud-context;3.1.8
org.springframework.cloud:spring-cloud-sleuth-api;3.1.10
org.springframework.cloud:spring-cloud-starter-sleuth;3.1.10
org.springframework.cloud:spring-cloud-starter-stream-kafka;3.2.10
org.springframework.cloud:spring-cloud-stream-binder-test;3.2.10
org.springframework.cloud:spring-cloud-stream;3.2.10

## Test dependency versions
com.github.docker-java:docker-java;3.4.0
Expand All @@ -189,7 +115,6 @@ com.microsoft.azure:azure-mgmt-graph-rbac;1.3.0
com.microsoft.azure:azure-mgmt-keyvault;1.11.1
com.microsoft.azure:azure-mgmt-network;1.33.0
com.microsoft.azure:azure-mgmt-resources;1.3.0
com.microsoft.azure:azure-mgmt-search;1.24.1
com.microsoft.azure:azure-mgmt-storage;1.3.0
com.microsoft.azure:azure-storage;8.0.0
com.microsoft.azure:msal4j;1.17.2
Expand Down Expand Up @@ -219,7 +144,6 @@ org.assertj:assertj-core;3.22.0
org.bouncycastle:bcprov-jdk15to18;1.78.1
org.bouncycastle:bcprov-jdk18on;1.78.1
org.bouncycastle:bcpkix-lts8on;2.73.6
org.eclipse.jetty:jetty-http;9.4.56.v20240826
org.eclipse.jetty:jetty-server;9.4.56.v20240826
org.eclipse.jetty:jetty-servlet;9.4.56.v20240826
org.hamcrest:hamcrest;2.2
Expand Down Expand Up @@ -279,7 +203,6 @@ org.apache.maven.plugins:maven-surefire-plugin;3.5.1
org.codehaus.mojo:animal-sniffer-maven-plugin;1.24
org.codehaus.mojo:build-helper-maven-plugin;3.6.0
org.codehaus.mojo:exec-maven-plugin;3.4.1
org.codehaus.mojo:properties-maven-plugin;1.2.1
org.codehaus.mojo:xml-maven-plugin;1.1.0
org.eclipse.jetty:jetty-maven-plugin;9.4.33.v20201020
org.eclipse.m2e:lifecycle-mapping;1.0.0
Expand Down Expand Up @@ -313,17 +236,12 @@ test_jar_com.microsoft.azure:azure-mgmt-resources;1.3.1-SNAPSHOT

# everything under sdk\cosmos
cosmos_com.google.guava:guava;33.0.0-jre
cosmos_com.microsoft.data.encryption:cryptography;0.2.2.jre8-preview
cosmos_com.microsoft.data.encryption:azure-key-vault-keystoreprovider;0.2.2.jre8-preview
cosmos_io.dropwizard.metrics:metrics-core;4.1.0
cosmos_io.dropwizard.metrics:metrics-graphite;4.1.0
cosmos_io.dropwizard.metrics:metrics-jvm;4.1.0
cosmos_org.mpierce.metrics.reservoir:hdrhistogram-metrics-reservoir;1.1.0
cosmos_org.hdrhistogram:HdrHistogram;2.1.12
cosmos_org.codehaus.mojo:build-helper-maven-plugin;3.5.0
cosmos_com.fasterxml.jackson.datatype:jackson-datatype-jsr310;2.15.2
cosmos_com.fasterxml.jackson.core:jackson-databind;2.15.2
cosmos_com.fasterxml.jackson.module:jackson-module-afterburner;2.15.2
cosmos_com.fasterxml.jackson.module:jackson-module-scala_2.12;2.15.2

## Cosmos Spark connector under sdk\cosmos\azure-cosmos-spark_3-1_2-12\pom.xml
Expand Down Expand Up @@ -369,7 +287,6 @@ cosmos_org.sourcelab:kafka-connect-client;4.0.4
cosmos_io.confluent:kafka-avro-serializer;7.6.0
cosmos_org.apache.avro:avro;1.11.3
# Maven Tools for Cosmos Kafka connector only
cosmos_io.confluent:kafka-connect-maven-plugin;0.12.0

# sdk\core\azure-core-serializer-avro-jackson\pom.xml
# This dependency is needed since Jackson Avro uses an older dependency on Apache Avro which is another library.
Expand Down Expand Up @@ -438,7 +355,6 @@ springboot3_org.apache.maven.plugins:maven-failsafe-plugin;3.5.2
springboot3_org.apache.maven.plugins:maven-jar-plugin;3.4.2
springboot3_org.apache.maven.plugins:maven-javadoc-plugin;3.10.1
springboot3_org.apache.maven.plugins:maven-surefire-plugin;3.5.2
springboot3_org.apache.qpid:qpid-jms-client;2.0.0
springboot3_org.assertj:assertj-core;3.26.3
springboot3_org.hibernate.validator:hibernate-validator;8.0.1.Final
springboot3_org.javatuples:javatuples;1.2
Expand All @@ -455,7 +371,6 @@ springboot3_org.springframework.boot:spring-boot-actuator;3.4.0
springboot3_org.springframework.boot:spring-boot-autoconfigure;3.4.0
springboot3_org.springframework.boot:spring-boot-configuration-metadata;3.4.0
springboot3_org.springframework.boot:spring-boot-configuration-processor;3.4.0
springboot3_org.springframework.boot:spring-boot-dependencies;3.4.0
springboot3_org.springframework.boot:spring-boot-docker-compose;3.4.0
springboot3_org.springframework.boot:spring-boot-starter-actuator;3.4.0
springboot3_org.springframework.boot:spring-boot-starter-data-jdbc;3.4.0
Expand All @@ -468,7 +383,6 @@ springboot3_org.springframework.boot:spring-boot-test;3.4.0
springboot3_org.springframework.boot:spring-boot-testcontainers;3.4.0
springboot3_org.springframework.cloud:spring-cloud-bus;4.2.0
springboot3_org.springframework.cloud:spring-cloud-context;4.2.0
springboot3_org.springframework.cloud:spring-cloud-dependencies;2024.0.0
springboot3_org.springframework.cloud:spring-cloud-starter-bootstrap;4.2.0
springboot3_org.springframework.cloud:spring-cloud-starter-stream-kafka;4.2.0
springboot3_org.springframework.cloud:spring-cloud-stream-test-binder;4.2.0
Expand Down Expand Up @@ -496,7 +410,6 @@ springboot3_org.springframework:spring-test;6.2.0
springboot3_org.springframework:spring-tx;6.2.0
springboot3_org.springframework:spring-web;6.2.0
springboot3_org.springframework:spring-webmvc;6.2.0
springboot3_org.testcontainers:testcontainers;1.20.4
springboot3_org.testcontainers:junit-jupiter;1.20.4
springboot3_org.testcontainers:azure;1.20.4

Expand Down
63 changes: 63 additions & 0 deletions eng/versioning/find_unused_dependencies.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.

# This script is used to find unused dependencies in the version_client.txt and external_dependencies.txt files.
# It is used in the CI pipeline to ensure that all dependencies are used in the codebase.

import argparse
import os

from utils import load_version_map_from_file
from utils import version_update_marker

def fixup_version_map(version_file, version_map):
# uses the util function to load the version map from the file, then adds a bool to each entry to track if it is visisted
load_version_map_from_file(version_file, version_map)
for key in version_map:
val = version_map[key]
version_map[key] = (False, val)

def find_unused_dependencies(dep_map, message):
unused_deps = [key for key in dep_map if not dep_map[key][0]]
if unused_deps:
print(message)
for dep in unused_deps:
print(" " + dep)
return bool(unused_deps)

def main():
version_map = {}
ext_dep_map = {}

version_file = os.path.normpath("eng/versioning/version_client.txt")
dependency_file = os.path.normpath("eng/versioning/external_dependencies.txt")

fixup_version_map(version_file, version_map)
fixup_version_map(dependency_file, ext_dep_map)


for root, _, files in os.walk("."):
try:
for file in files:
if file.startswith("pom") and file.endswith(".xml"):
with open(os.path.join(root, file), encoding="utf-8") as f:
for line in f:
match = version_update_marker.search(line)
if match:
module_name, version_type = match.group(1), match.group(2)
if module_name in ext_dep_map or module_name in version_map:
if version_type == "external_dependency":
ext_dep_map[module_name] = (True, ext_dep_map[module_name][1])
else:
version_map[module_name] = (True, version_map[module_name][1])
except KeyError as e:
print(str(e) + " was not found in the right place. Please investigate.")

unused_dependencies = find_unused_dependencies(version_map, "Unused version_client.txt entries:")
unused_ext_dep = find_unused_dependencies(ext_dep_map, "Unused external_dependencies.txt entries:")

if unused_dependencies or unused_ext_dep:
exit(1)

if __name__ == "__main__":
main()
24 changes: 1 addition & 23 deletions eng/versioning/update_versions.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import traceback
from utils import BuildType
from utils import CodeModule
from utils import load_version_map_from_file
from utils import external_dependency_version_regex
from utils import external_dependency_include_regex
from utils import run_check_call
Expand Down Expand Up @@ -185,29 +186,6 @@ def update_changelog(pom_file, is_increment, library_array):
else:
print('There is no CHANGELOG.md file in {}, skipping update'.format(dirname))

def load_version_map_from_file(the_file, version_map):
with open(the_file) as f:
for raw_line in f:
stripped_line = raw_line.strip()
if not stripped_line or stripped_line.startswith('#'):
continue
module = CodeModule(stripped_line)
# verify no duplicate entries
if (module.name in version_map):
raise ValueError('Version file: {0} contains a duplicate entry: {1}'.format(the_file, module.name))
# verify that if the module is beta_ or unreleased_ that there's a matching non-beta_ or non-unreleased_ entry
if (module.name.startswith('beta_') or module.name.startswith('unreleased_')):
tempName = module.name
if tempName.startswith('beta_'):
tempName = module.name[len('beta_'):]
else:
tempName = module.name[len('unreleased_'):]
# if there isn't a non beta or unreleased entry then raise an issue
if tempName not in version_map:
raise ValueError('Version file: {0} does not contain a non-beta or non-unreleased entry for beta_/unreleased_ library: {1}'.format(the_file, module.name))

version_map[module.name] = module

def load_version_overrides(the_file, version_map, overrides_name):
with open(the_file) as f:
data = json.load(f)
Expand Down
26 changes: 25 additions & 1 deletion eng/versioning/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -156,4 +156,28 @@ def run_check_call(
if always_exit:
exit(1)
else:
return err
return err


def load_version_map_from_file(the_file, version_map):
with open(the_file) as f:
for raw_line in f:
stripped_line = raw_line.strip()
if not stripped_line or stripped_line.startswith('#'):
continue
module = CodeModule(stripped_line)
# verify no duplicate entries
if (module.name in version_map):
raise ValueError('Version file: {0} contains a duplicate entry: {1}'.format(the_file, module.name))
# verify that if the module is beta_ or unreleased_ that there's a matching non-beta_ or non-unreleased_ entry
if (module.name.startswith('beta_') or module.name.startswith('unreleased_')):
tempName = module.name
if tempName.startswith('beta_'):
tempName = module.name[len('beta_'):]
else:
tempName = module.name[len('unreleased_'):]
# if there isn't a non beta or unreleased entry then raise an issue
if tempName not in version_map:
raise ValueError('Version file: {0} does not contain a non-beta or non-unreleased entry for beta_/unreleased_ library: {1}'.format(the_file, module.name))
version_map[module.name] = module

Loading

0 comments on commit d2f8f7f

Please sign in to comment.