From 06f89a3f015bd896257995cef3ca8362a773cb46 Mon Sep 17 00:00:00 2001 From: Tigran Manasyan Date: Tue, 24 Sep 2024 18:41:03 +0400 Subject: [PATCH 1/3] [ADH-5065] Add metrics support --- conf/smart-default.xml | 46 +++++ lombok.config | 1 + pom.xml | 13 +- smart-agent/pom.xml | 17 ++ .../java/org/smartdata/agent/SmartAgent.java | 168 +++++++++-------- .../org.smartdata.agent/TestSmartAgent.java | 8 +- smart-common/pom.xml | 5 - .../main/java/org/smartdata/SmartContext.java | 11 +- .../org/smartdata/conf/SmartConfKeys.java | 29 +++ .../smartdata/metrics/DataAccessEvent.java | 0 .../smartdata/metrics/FileAccessEvent.java | 28 +-- .../metrics/FileAccessEventCollector.java | 0 .../metrics/FileAccessEventSource.java | 0 .../impl/FileAccessMetricsFactory.java | 12 +- .../impl/SmartServerAccessEventCollector.java | 0 .../impl/SmartServerAccessEventSource.java | 30 +-- .../security/NoOpSmartPrincipalManager.java | 35 ++++ .../security/SmartPrincipalManager.java | 4 + smart-dist/pom.xml | 5 - .../org/smartdata/server/SmartEngine.java | 4 +- .../server/engine/CmdletManager.java | 15 +- .../server/engine/ServerContext.java | 16 +- .../server/engine/StatesManager.java | 14 +- .../engine/cmdlet/CmdletDispatcher.java | 7 +- .../engine/cmdlet/CmdletManagerContext.java | 10 +- .../engine/data/AccessEventFetcher.java | 13 +- .../smart-hadoop-client-3/pom.xml | 6 - .../smartdata/hdfs/client/SmartDFSClient.java | 1 - .../metric/NNMetricsAccessEventCollector.java | 1 - smart-integration/pom.xml | 2 +- .../integration/TestRuleRestApi.java | 2 +- .../org/smartdata/integration/TestSsl.java | 10 +- .../integration/auth/TestWebServerAuth.java | 2 +- .../auth/TestWebServerLdapAuth.java | 38 ++-- .../TestWebServerPredefinedUsersAuth.java | 6 +- smart-metastore/pom.xml | 5 + .../java/org/smartdata/metastore/DBPool.java | 6 + .../org/smartdata/metastore/DruidPool.java | 11 +- .../metastore/DruidPoolMetricsBinder.java | 173 ++++++++++++++++++ .../org/smartdata/metastore/MetaStore.java | 6 +- .../metastore/utils/MetaStoreUtils.java | 2 +- .../org/smartdata/metastore/TestDaoBase.java | 3 +- .../org/smartdata/metastore/TestDruid.java | 3 +- smart-metrics/README.md | 69 +++++++ smart-metrics/pom.xml | 49 +++-- .../metrics/CompositeMetricsFactory.java | 162 ++++++++++++++++ .../org/smartdata/metrics/MetricsFactory.java | 47 +++++ .../metrics/MetricsFactoryHolder.java | 14 +- .../smartdata/metrics/NoOpMetricsFactory.java | 62 +++++++ smart-server/pom.xml | 7 +- .../org/smartdata/server/SmartServer.java | 17 +- .../resources/hadoop-metrics2-ssm.properties | 5 + .../server/engine/TestCmdletManager.java | 2 +- smart-web-server/pom.xml | 138 ++------------ .../smart-agent-web-server/pom.xml | 40 ++++ .../agent/http/SmartAgentHttpServer.java | 65 +++++++ .../src/main/resources/application-agent.yaml | 5 + .../.openapi-generator-ignore | 0 .../smart-master-web-server/pom.xml | 171 +++++++++++++++++ .../server/SmartMasterRestServer.java} | 42 ++++- .../controller/ActionsControllerDelegate.java | 0 .../controller/AuditControllerDelegate.java | 0 .../CachedFilesControllerDelegate.java | 0 .../controller/ClusterControllerDelegate.java | 0 .../controller/CmdletControllerDelegate.java | 0 .../controller/FilesControllerDelegate.java | 0 .../controller/RulesControllerDelegate.java | 0 .../controller/SystemControllerDelegate.java | 0 .../server/controller/WebUiController.java | 0 .../server/generated/api/ActionsApi.java | 0 .../generated/api/ActionsApiController.java | 0 .../generated/api/ActionsApiDelegate.java | 0 .../server/generated/api/AuditApi.java | 0 .../generated/api/AuditApiController.java | 0 .../generated/api/AuditApiDelegate.java | 0 .../server/generated/api/ClusterApi.java | 0 .../generated/api/ClusterApiController.java | 0 .../generated/api/ClusterApiDelegate.java | 0 .../server/generated/api/CmdletsApi.java | 0 .../generated/api/CmdletsApiController.java | 0 .../generated/api/CmdletsApiDelegate.java | 0 .../server/generated/api/FilesApi.java | 0 .../generated/api/FilesApiController.java | 0 .../generated/api/FilesApiDelegate.java | 0 .../server/generated/api/RulesApi.java | 0 .../generated/api/RulesApiController.java | 0 .../generated/api/RulesApiDelegate.java | 0 .../server/generated/api/SystemApi.java | 0 .../generated/api/SystemApiController.java | 0 .../generated/api/SystemApiDelegate.java | 0 .../config/EnumConverterConfiguration.java | 0 .../server/generated/model/ActionDto.java | 0 .../server/generated/model/ActionInfoDto.java | 0 .../server/generated/model/ActionSortDto.java | 0 .../generated/model/ActionSourceDto.java | 0 .../generated/model/ActionStateDto.java | 0 .../server/generated/model/ActionsDto.java | 0 .../server/generated/model/AuditEventDto.java | 0 .../generated/model/AuditEventResultDto.java | 0 .../generated/model/AuditEventsDto.java | 0 .../generated/model/AuditObjectTypeDto.java | 0 .../generated/model/AuditOperationDto.java | 0 .../server/generated/model/AuditSortDto.java | 0 .../generated/model/CachedFileInfoDto.java | 0 .../generated/model/CachedFileSortDto.java | 0 .../generated/model/CachedFilesDto.java | 0 .../model/CachedTimeIntervalDto.java | 0 .../generated/model/ClusterNodeDto.java | 0 .../generated/model/ClusterNodesDto.java | 0 .../generated/model/ClusterSortDto.java | 0 .../server/generated/model/CmdletDto.java | 0 .../server/generated/model/CmdletSortDto.java | 0 .../generated/model/CmdletStateDto.java | 0 .../server/generated/model/CmdletsDto.java | 0 .../model/CompletionTimeIntervalDto.java | 0 .../generated/model/ErrorResponseDto.java | 0 .../generated/model/EventTimeIntervalDto.java | 0 .../generated/model/ExecutorTypeDto.java | 0 .../generated/model/FileAccessCountsDto.java | 0 .../generated/model/FileAccessInfoDto.java | 0 .../generated/model/HotFileSortDto.java | 0 .../model/LastAccessedTimeIntervalDto.java | 0 .../model/LastActivationTimeIntervalDto.java | 0 .../generated/model/PageRequestDto.java | 0 .../model/RegistrationTimeIntervalDto.java | 0 .../server/generated/model/RuleDto.java | 0 .../server/generated/model/RuleSortDto.java | 0 .../server/generated/model/RuleStateDto.java | 0 .../server/generated/model/RulesDto.java | 0 .../server/generated/model/RulesInfoDto.java | 0 .../model/StateChangeTimeIntervalDto.java | 0 .../model/SubmissionTimeIntervalDto.java | 0 .../model/SubmitActionRequestDto.java | 0 .../model/SubmitCmdletRequestDto.java | 0 .../generated/model/SubmitRuleRequestDto.java | 0 .../server/generated/model/UserInfoDto.java | 0 .../server/mappers/ActionInfoMapper.java | 0 .../server/mappers/AuditEventMapper.java | 0 .../server/mappers/CachedFileMapper.java | 0 .../server/mappers/ClusterNodeMapper.java | 0 .../server/mappers/CmdletInfoMapper.java | 0 .../server/mappers/FileAccessInfoMapper.java | 0 .../server/mappers/RuleInfoMapper.java | 0 .../smartdata/server/mappers/SmartMapper.java | 0 .../pagination/ActionPageRequestMapper.java | 0 .../pagination/AuditPageRequestMapper.java | 0 .../pagination/BasePageRequestMapper.java | 0 .../CachedFilePageRequestMapper.java | 0 .../ClusterNodePageRequestMapper.java | 0 .../pagination/CmdletsPageRequestMapper.java | 0 .../FileAccessInfoPageRequestMapper.java | 0 .../pagination/RulesPageRequestMapper.java | 0 .../api/parameters/actions/ActionSort.yaml | 0 .../api/parameters/actions/action-sort.yaml | 0 .../api/parameters/actions/action-states.yaml | 0 .../parameters/actions/completion-time.yaml | 0 .../api/parameters/actions/hosts.yaml | 0 .../api/parameters/actions/sources.yaml | 0 .../api/parameters/audit/AuditSort.yaml | 0 .../api/parameters/audit/audit-sort.yaml | 0 .../api/parameters/audit/event-time.yaml | 0 .../api/parameters/audit/object-ids.yaml | 0 .../api/parameters/audit/object-types.yaml | 0 .../api/parameters/audit/operations.yaml | 0 .../api/parameters/audit/results.yaml | 0 .../api/parameters/audit/username-like.yaml | 0 .../api/parameters/cluster/ClusterSort.yaml | 0 .../api/parameters/cluster/cluster-sort.yaml | 0 .../parameters/cluster/registration-time.yaml | 0 .../api/parameters/cmdlets/CmdletSort.yaml | 0 .../api/parameters/cmdlets/cmdlets-sort.yaml | 0 .../api/parameters/cmdlets/rule-ids.yaml | 0 .../cmdlets/state-changed-time.yaml | 0 .../api/parameters/cmdlets/states.yaml | 0 .../api/parameters/common/PageRequest.yaml | 0 .../resources/api/parameters/common/id.yaml | 0 .../api/parameters/common/page-request.yaml | 0 .../parameters/common/submission-time.yaml | 0 .../common/text-representation-like.yaml | 0 .../api/parameters/files/CachedFileSort.yaml | 0 .../api/parameters/files/HotFileSort.yaml | 0 .../parameters/files/cached-files-sort.yaml | 0 .../api/parameters/files/cached-time.yaml | 0 .../api/parameters/files/hot-files-sort.yaml | 0 .../parameters/files/last-accessed-time.yaml | 0 .../api/parameters/files/path-like.yaml | 0 .../api/parameters/rules/RuleSort.yaml | 0 .../rules/last-activation-time.yaml | 0 .../api/parameters/rules/rule-states.yaml | 0 .../api/parameters/rules/rules-sort.yaml | 0 .../main/resources/api/resources/action.yaml | 0 .../main/resources/api/resources/actions.yaml | 0 .../resources/api/resources/audit-events.yaml | 0 .../api/resources/cluster-nodes.yaml | 0 .../resources/api/resources/cmdlet-stop.yaml | 0 .../main/resources/api/resources/cmdlet.yaml | 0 .../main/resources/api/resources/cmdlets.yaml | 0 .../resources/api/resources/current-user.yaml | 0 .../api/resources/files-access-counts.yaml | 0 .../resources/api/resources/files-cached.yaml | 0 .../resources/api/resources/rule-start.yaml | 0 .../resources/api/resources/rule-stop.yaml | 0 .../main/resources/api/resources/rule.yaml | 0 .../resources/api/resources/rules-info.yaml | 0 .../main/resources/api/resources/rules.yaml | 0 .../resources/api/schemas/actions/Action.yaml | 0 .../api/schemas/actions/ActionInfo.yaml | 0 .../api/schemas/actions/ActionSource.yaml | 0 .../api/schemas/actions/ActionState.yaml | 0 .../api/schemas/audit/AuditEvent.yaml | 0 .../api/schemas/audit/AuditEventResult.yaml | 0 .../api/schemas/audit/AuditObjectType.yaml | 0 .../api/schemas/audit/AuditOperation.yaml | 0 .../api/schemas/cluster/ClusterNode.yaml | 0 .../api/schemas/cluster/ExecutorType.yaml | 0 .../resources/api/schemas/cmdlets/Cmdlet.yaml | 0 .../api/schemas/cmdlets/CmdletState.yaml | 0 .../api/schemas/common/ErrorResponse.yaml | 0 .../api/schemas/common/PageResponse.yaml | 0 .../api/schemas/files/CachedFileInfo.yaml | 0 .../api/schemas/files/FileAccessInfo.yaml | 0 .../schemas/requests/SubmitActionRequest.yaml | 0 .../schemas/requests/SubmitCmdletRequest.yaml | 0 .../schemas/requests/SubmitRuleRequest.yaml | 0 .../resources/api/schemas/rules/Rule.yaml | 0 .../api/schemas/rules/RuleState.yaml | 0 .../api/schemas/rules/RulesInfo.yaml | 0 .../api/schemas/system/UserInfo.yaml | 0 .../src/main/resources/api/ssm-api.yaml | 0 .../main/resources/application-master.yaml} | 5 + .../resources/static/.openapi-generator/FILES | 0 .../static/.openapi-generator/VERSION | 0 ...m-api.yaml-generate-merged-api-spec.sha256 | 0 .../src/main/resources/static/ssm-api.yaml | 0 .../smart-web-server-base/pom.xml | 33 ++++ .../org/smartdata/http/SmartHttpServer.java} | 59 ++++-- .../smartdata/http}/config/ConfigKeys.java | 6 +- .../config/EmbeddedServerCustomizer.java | 23 +-- ...erberosBasicAuthSecurityConfiguration.java | 6 +- .../config/LdapAuthSecurityConfiguration.java | 30 +-- .../http}/config/PasswordEncoderFactory.java | 2 +- .../PredefinedUsersSecurityConfiguration.java | 14 +- .../http}/config/SecurityConfiguration.java | 11 +- .../config/SpnegoSecurityConfiguration.java | 10 +- .../http}/config/SsmAuthHttpConfigurer.java | 2 +- ...dditionalFilterTemplateFactoryWrapper.java | 8 +- .../config/ldap/search/LdapSearchScope.java | 2 +- .../search/LdapSearchTemplateFactory.java | 4 +- .../ldap/search/LdapUserSearchFactory.java | 26 +-- .../http}/config/ldap/search/LdapUtils.java | 8 +- .../group/GroupSearchByMemberAttrFactory.java | 14 +- .../group/GroupSearchByNameAttrFactory.java | 24 +-- .../ldap/search/group/GroupSearchRunner.java | 22 +-- .../ldap/search/group/SsmLdapGroupSearch.java | 4 +- .../search/query/LdapExpressionTemplate.java | 2 +- .../config/ldap/search/query/LdapFilter.java | 2 +- .../ldap/search/query/LdapOperator.java | 2 +- .../ldap/search/query/LdapQueryDsl.java | 2 +- .../search/user/UserGroupSearchRunner.java | 6 +- .../user/UserSearchByCustomQueryFactory.java | 8 +- .../UserSearchByMembershipAttrFactory.java | 18 +- .../UserSearchByNameAttributeFactory.java | 20 +- .../ldap/search/user/UserSearchRunner.java | 18 +- .../config/metrics/MetricsConfiguration.java | 41 +++++ .../PrometheusMetricsConfiguration.java | 57 ++++++ .../error/AuthenticationFailureListener.java | 0 .../org/smartdata/http}/error/ErrorDto.java | 2 +- .../http}/error/SmartExceptionHandler.java | 46 ++--- .../smartdata/http}/error/SsmErrorCode.java | 2 +- .../SmartPrincipalInitializerFilter.java | 2 +- .../org/smartdata/http}/util/ConfigUtils.java | 4 +- ...dditionalFilterTemplateFactoryWrapper.java | 10 +- .../TestGroupSearchByMemberAttrFactory.java | 10 +- .../TestGroupSearchByNameAttrFactory.java | 10 +- .../search/TestLdapSearchTemplateFactory.java | 2 +- .../TestUserSearchByCustomQueryFactory.java | 6 +- ...TestUserSearchByMembershipAttrFactory.java | 14 +- .../TestUserSearchByNameAttrFactory.java | 8 +- .../smartdata/http}/util/ConfigUtilsTest.java | 6 +- .../docker/multihost/conf/prometheus.yml | 18 ++ .../docker/multihost/conf/smart-site.xml | 2 +- .../docker/multihost/docker-compose.yaml | 12 ++ .../docker/multihost/kerberos/kdc-init.sh | 2 + 283 files changed, 1677 insertions(+), 631 deletions(-) rename {smart-metrics => smart-common}/src/main/java/org/smartdata/metrics/DataAccessEvent.java (100%) rename {smart-metrics => smart-common}/src/main/java/org/smartdata/metrics/FileAccessEvent.java (79%) rename {smart-metrics => smart-common}/src/main/java/org/smartdata/metrics/FileAccessEventCollector.java (100%) rename {smart-metrics => smart-common}/src/main/java/org/smartdata/metrics/FileAccessEventSource.java (100%) rename smart-metrics/src/main/java/org/smartdata/metrics/impl/MetricsFactory.java => smart-common/src/main/java/org/smartdata/metrics/impl/FileAccessMetricsFactory.java (71%) rename {smart-metrics => smart-common}/src/main/java/org/smartdata/metrics/impl/SmartServerAccessEventCollector.java (100%) rename {smart-metrics => smart-common}/src/main/java/org/smartdata/metrics/impl/SmartServerAccessEventSource.java (64%) create mode 100644 smart-common/src/main/java/org/smartdata/security/NoOpSmartPrincipalManager.java create mode 100644 smart-metastore/src/main/java/org/smartdata/metastore/DruidPoolMetricsBinder.java create mode 100644 smart-metrics/README.md create mode 100644 smart-metrics/src/main/java/org/smartdata/metrics/CompositeMetricsFactory.java create mode 100644 smart-metrics/src/main/java/org/smartdata/metrics/MetricsFactory.java rename smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/user/SsmLdapUserSearch.java => smart-metrics/src/main/java/org/smartdata/metrics/MetricsFactoryHolder.java (72%) create mode 100644 smart-metrics/src/main/java/org/smartdata/metrics/NoOpMetricsFactory.java create mode 100644 smart-server/src/main/resources/hadoop-metrics2-ssm.properties create mode 100644 smart-web-server/smart-agent-web-server/pom.xml create mode 100644 smart-web-server/smart-agent-web-server/src/main/java/org/smartdata/agent/http/SmartAgentHttpServer.java create mode 100644 smart-web-server/smart-agent-web-server/src/main/resources/application-agent.yaml rename smart-web-server/{ => smart-master-web-server}/.openapi-generator-ignore (100%) create mode 100644 smart-web-server/smart-master-web-server/pom.xml rename smart-web-server/{src/main/java/org/smartdata/server/config/SsmContextInitializer.java => smart-master-web-server/src/main/java/org/smartdata/server/SmartMasterRestServer.java} (64%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/controller/ActionsControllerDelegate.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/controller/AuditControllerDelegate.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/controller/CachedFilesControllerDelegate.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/controller/ClusterControllerDelegate.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/controller/CmdletControllerDelegate.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/controller/FilesControllerDelegate.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/controller/RulesControllerDelegate.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/controller/SystemControllerDelegate.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/controller/WebUiController.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/api/ActionsApi.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/api/ActionsApiController.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/api/ActionsApiDelegate.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/api/AuditApi.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/api/AuditApiController.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/api/AuditApiDelegate.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/api/ClusterApi.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/api/ClusterApiController.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/api/ClusterApiDelegate.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/api/CmdletsApi.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/api/CmdletsApiController.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/api/CmdletsApiDelegate.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/api/FilesApi.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/api/FilesApiController.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/api/FilesApiDelegate.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/api/RulesApi.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/api/RulesApiController.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/api/RulesApiDelegate.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/api/SystemApi.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/api/SystemApiController.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/api/SystemApiDelegate.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/config/EnumConverterConfiguration.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/ActionDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/ActionInfoDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/ActionSortDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/ActionSourceDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/ActionStateDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/ActionsDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/AuditEventDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/AuditEventResultDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/AuditEventsDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/AuditObjectTypeDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/AuditOperationDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/AuditSortDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/CachedFileInfoDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/CachedFileSortDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/CachedFilesDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/CachedTimeIntervalDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/ClusterNodeDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/ClusterNodesDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/ClusterSortDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/CmdletDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/CmdletSortDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/CmdletStateDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/CmdletsDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/CompletionTimeIntervalDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/ErrorResponseDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/EventTimeIntervalDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/ExecutorTypeDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/FileAccessCountsDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/FileAccessInfoDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/HotFileSortDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/LastAccessedTimeIntervalDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/LastActivationTimeIntervalDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/PageRequestDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/RegistrationTimeIntervalDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/RuleDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/RuleSortDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/RuleStateDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/RulesDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/RulesInfoDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/StateChangeTimeIntervalDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/SubmissionTimeIntervalDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/SubmitActionRequestDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/SubmitCmdletRequestDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/SubmitRuleRequestDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/generated/model/UserInfoDto.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/mappers/ActionInfoMapper.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/mappers/AuditEventMapper.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/mappers/CachedFileMapper.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/mappers/ClusterNodeMapper.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/mappers/CmdletInfoMapper.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/mappers/FileAccessInfoMapper.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/mappers/RuleInfoMapper.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/mappers/SmartMapper.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/mappers/pagination/ActionPageRequestMapper.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/mappers/pagination/AuditPageRequestMapper.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/mappers/pagination/BasePageRequestMapper.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/mappers/pagination/CachedFilePageRequestMapper.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/mappers/pagination/ClusterNodePageRequestMapper.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/mappers/pagination/CmdletsPageRequestMapper.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/mappers/pagination/FileAccessInfoPageRequestMapper.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/java/org/smartdata/server/mappers/pagination/RulesPageRequestMapper.java (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/parameters/actions/ActionSort.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/parameters/actions/action-sort.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/parameters/actions/action-states.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/parameters/actions/completion-time.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/parameters/actions/hosts.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/parameters/actions/sources.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/parameters/audit/AuditSort.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/parameters/audit/audit-sort.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/parameters/audit/event-time.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/parameters/audit/object-ids.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/parameters/audit/object-types.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/parameters/audit/operations.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/parameters/audit/results.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/parameters/audit/username-like.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/parameters/cluster/ClusterSort.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/parameters/cluster/cluster-sort.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/parameters/cluster/registration-time.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/parameters/cmdlets/CmdletSort.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/parameters/cmdlets/cmdlets-sort.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/parameters/cmdlets/rule-ids.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/parameters/cmdlets/state-changed-time.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/parameters/cmdlets/states.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/parameters/common/PageRequest.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/parameters/common/id.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/parameters/common/page-request.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/parameters/common/submission-time.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/parameters/common/text-representation-like.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/parameters/files/CachedFileSort.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/parameters/files/HotFileSort.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/parameters/files/cached-files-sort.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/parameters/files/cached-time.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/parameters/files/hot-files-sort.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/parameters/files/last-accessed-time.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/parameters/files/path-like.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/parameters/rules/RuleSort.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/parameters/rules/last-activation-time.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/parameters/rules/rule-states.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/parameters/rules/rules-sort.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/resources/action.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/resources/actions.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/resources/audit-events.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/resources/cluster-nodes.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/resources/cmdlet-stop.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/resources/cmdlet.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/resources/cmdlets.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/resources/current-user.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/resources/files-access-counts.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/resources/files-cached.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/resources/rule-start.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/resources/rule-stop.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/resources/rule.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/resources/rules-info.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/resources/rules.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/schemas/actions/Action.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/schemas/actions/ActionInfo.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/schemas/actions/ActionSource.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/schemas/actions/ActionState.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/schemas/audit/AuditEvent.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/schemas/audit/AuditEventResult.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/schemas/audit/AuditObjectType.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/schemas/audit/AuditOperation.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/schemas/cluster/ClusterNode.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/schemas/cluster/ExecutorType.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/schemas/cmdlets/Cmdlet.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/schemas/cmdlets/CmdletState.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/schemas/common/ErrorResponse.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/schemas/common/PageResponse.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/schemas/files/CachedFileInfo.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/schemas/files/FileAccessInfo.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/schemas/requests/SubmitActionRequest.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/schemas/requests/SubmitCmdletRequest.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/schemas/requests/SubmitRuleRequest.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/schemas/rules/Rule.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/schemas/rules/RuleState.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/schemas/rules/RulesInfo.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/schemas/system/UserInfo.yaml (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/api/ssm-api.yaml (100%) rename smart-web-server/{src/main/resources/application.yaml => smart-master-web-server/src/main/resources/application-master.yaml} (81%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/static/.openapi-generator/FILES (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/static/.openapi-generator/VERSION (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/static/.openapi-generator/ssm-api.yaml-generate-merged-api-spec.sha256 (100%) rename smart-web-server/{ => smart-master-web-server}/src/main/resources/static/ssm-api.yaml (100%) create mode 100644 smart-web-server/smart-web-server-base/pom.xml rename smart-web-server/{src/main/java/org/smartdata/server/SmartRestServer.java => smart-web-server-base/src/main/java/org/smartdata/http/SmartHttpServer.java} (54%) rename smart-web-server/{src/main/java/org/smartdata/server => smart-web-server-base/src/main/java/org/smartdata/http}/config/ConfigKeys.java (97%) rename smart-web-server/{src/main/java/org/smartdata/server => smart-web-server-base/src/main/java/org/smartdata/http}/config/EmbeddedServerCustomizer.java (73%) rename smart-web-server/{src/main/java/org/smartdata/server => smart-web-server-base/src/main/java/org/smartdata/http}/config/KerberosBasicAuthSecurityConfiguration.java (92%) rename smart-web-server/{src/main/java/org/smartdata/server => smart-web-server-base/src/main/java/org/smartdata/http}/config/LdapAuthSecurityConfiguration.java (79%) rename smart-web-server/{src/main/java/org/smartdata/server => smart-web-server-base/src/main/java/org/smartdata/http}/config/PasswordEncoderFactory.java (98%) rename smart-web-server/{src/main/java/org/smartdata/server => smart-web-server-base/src/main/java/org/smartdata/http}/config/PredefinedUsersSecurityConfiguration.java (83%) rename smart-web-server/{src/main/java/org/smartdata/server => smart-web-server-base/src/main/java/org/smartdata/http}/config/SecurityConfiguration.java (90%) rename smart-web-server/{src/main/java/org/smartdata/server => smart-web-server-base/src/main/java/org/smartdata/http}/config/SpnegoSecurityConfiguration.java (92%) rename smart-web-server/{src/main/java/org/smartdata/server => smart-web-server-base/src/main/java/org/smartdata/http}/config/SsmAuthHttpConfigurer.java (96%) rename smart-web-server/{src/main/java/org/smartdata/server => smart-web-server-base/src/main/java/org/smartdata/http}/config/ldap/search/AdditionalFilterTemplateFactoryWrapper.java (86%) rename smart-web-server/{src/main/java/org/smartdata/server => smart-web-server-base/src/main/java/org/smartdata/http}/config/ldap/search/LdapSearchScope.java (95%) rename smart-web-server/{src/main/java/org/smartdata/server => smart-web-server-base/src/main/java/org/smartdata/http}/config/ldap/search/LdapSearchTemplateFactory.java (87%) rename smart-web-server/{src/main/java/org/smartdata/server => smart-web-server-base/src/main/java/org/smartdata/http}/config/ldap/search/LdapUserSearchFactory.java (77%) rename smart-web-server/{src/main/java/org/smartdata/server => smart-web-server-base/src/main/java/org/smartdata/http}/config/ldap/search/LdapUtils.java (88%) rename smart-web-server/{src/main/java/org/smartdata/server => smart-web-server-base/src/main/java/org/smartdata/http}/config/ldap/search/group/GroupSearchByMemberAttrFactory.java (76%) rename smart-web-server/{src/main/java/org/smartdata/server => smart-web-server-base/src/main/java/org/smartdata/http}/config/ldap/search/group/GroupSearchByNameAttrFactory.java (69%) rename smart-web-server/{src/main/java/org/smartdata/server => smart-web-server-base/src/main/java/org/smartdata/http}/config/ldap/search/group/GroupSearchRunner.java (81%) rename smart-web-server/{src/main/java/org/smartdata/server => smart-web-server-base/src/main/java/org/smartdata/http}/config/ldap/search/group/SsmLdapGroupSearch.java (88%) rename smart-web-server/{src/main/java/org/smartdata/server => smart-web-server-base/src/main/java/org/smartdata/http}/config/ldap/search/query/LdapExpressionTemplate.java (95%) rename smart-web-server/{src/main/java/org/smartdata/server => smart-web-server-base/src/main/java/org/smartdata/http}/config/ldap/search/query/LdapFilter.java (95%) rename smart-web-server/{src/main/java/org/smartdata/server => smart-web-server-base/src/main/java/org/smartdata/http}/config/ldap/search/query/LdapOperator.java (97%) rename smart-web-server/{src/main/java/org/smartdata/server => smart-web-server-base/src/main/java/org/smartdata/http}/config/ldap/search/query/LdapQueryDsl.java (96%) rename smart-web-server/{src/main/java/org/smartdata/server => smart-web-server-base/src/main/java/org/smartdata/http}/config/ldap/search/user/UserGroupSearchRunner.java (93%) rename smart-web-server/{src/main/java/org/smartdata/server => smart-web-server-base/src/main/java/org/smartdata/http}/config/ldap/search/user/UserSearchByCustomQueryFactory.java (83%) rename smart-web-server/{src/main/java/org/smartdata/server => smart-web-server-base/src/main/java/org/smartdata/http}/config/ldap/search/user/UserSearchByMembershipAttrFactory.java (79%) rename smart-web-server/{src/main/java/org/smartdata/server => smart-web-server-base/src/main/java/org/smartdata/http}/config/ldap/search/user/UserSearchByNameAttributeFactory.java (69%) rename smart-web-server/{src/main/java/org/smartdata/server => smart-web-server-base/src/main/java/org/smartdata/http}/config/ldap/search/user/UserSearchRunner.java (84%) create mode 100644 smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/metrics/MetricsConfiguration.java create mode 100644 smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/metrics/PrometheusMetricsConfiguration.java rename smart-web-server/{src/main/java/org/smartdata/server => smart-web-server-base/src/main/java/org/smartdata/http}/error/AuthenticationFailureListener.java (100%) rename smart-web-server/{src/main/java/org/smartdata/server => smart-web-server-base/src/main/java/org/smartdata/http}/error/ErrorDto.java (96%) rename smart-web-server/{src/main/java/org/smartdata/server => smart-web-server-base/src/main/java/org/smartdata/http}/error/SmartExceptionHandler.java (86%) rename smart-web-server/{src/main/java/org/smartdata/server => smart-web-server-base/src/main/java/org/smartdata/http}/error/SsmErrorCode.java (96%) rename smart-web-server/{src/main/java/org/smartdata/server => smart-web-server-base/src/main/java/org/smartdata/http}/security/SmartPrincipalInitializerFilter.java (98%) rename smart-web-server/{src/main/java/org/smartdata/server => smart-web-server-base/src/main/java/org/smartdata/http}/util/ConfigUtils.java (95%) rename smart-web-server/{src/test/java/org/smartdata/server => smart-web-server-base/src/test/java/org/smartdata/http}/config/ldap/search/TestAdditionalFilterTemplateFactoryWrapper.java (82%) rename smart-web-server/{src/test/java/org/smartdata/server => smart-web-server-base/src/test/java/org/smartdata/http}/config/ldap/search/TestGroupSearchByMemberAttrFactory.java (83%) rename smart-web-server/{src/test/java/org/smartdata/server => smart-web-server-base/src/test/java/org/smartdata/http}/config/ldap/search/TestGroupSearchByNameAttrFactory.java (83%) rename smart-web-server/{src/test/java/org/smartdata/server => smart-web-server-base/src/test/java/org/smartdata/http}/config/ldap/search/TestLdapSearchTemplateFactory.java (96%) rename smart-web-server/{src/test/java/org/smartdata/server => smart-web-server-base/src/test/java/org/smartdata/http}/config/ldap/search/TestUserSearchByCustomQueryFactory.java (86%) rename smart-web-server/{src/test/java/org/smartdata/server => smart-web-server-base/src/test/java/org/smartdata/http}/config/ldap/search/TestUserSearchByMembershipAttrFactory.java (83%) rename smart-web-server/{src/test/java/org/smartdata/server => smart-web-server-base/src/test/java/org/smartdata/http}/config/ldap/search/TestUserSearchByNameAttrFactory.java (84%) rename smart-web-server/{src/test/java/org/smartdata/server => smart-web-server-base/src/test/java/org/smartdata/http}/util/ConfigUtilsTest.java (94%) create mode 100644 supports/tools/docker/multihost/conf/prometheus.yml diff --git a/conf/smart-default.xml b/conf/smart-default.xml index 8a9b641cb2c..cdcec656b81 100644 --- a/conf/smart-default.xml +++ b/conf/smart-default.xml @@ -457,6 +457,12 @@ SSM Rest Server port + + smart.agent.http.server.port + 8081 + SSM Agent HTTP server port + + smart.client.report.tasks.timeout.ms 2000 @@ -724,4 +730,44 @@ The threshold number of file diff cache elements for starting cache invalidation. + + + smart.metrics.enabled + true + + Whether to enable metrics collection and export. + + + + + smart.metrics.jmx.enabled + true + + Whether to enable JMX metrics exporter. + + + + + smart.metrics.jmx.domain + metrics + + Default domain for JMX exporter. + + + + + smart.metrics.prometheus.enabled + true + + Whether to enable Prometheus metrics exporter. + + + + + smart.metrics.db.queries.enabled + true + + Whether to enable SQL queries statistics export. + + diff --git a/lombok.config b/lombok.config index 5158a350de9..8a3cdf4ea28 100644 --- a/lombok.config +++ b/lombok.config @@ -1,2 +1,3 @@ config.stopBubbling = true lombok.builder.classname = Builder +lombok.copyableAnnotations += org.springframework.beans.factory.annotation.Qualifier diff --git a/pom.xml b/pom.xml index 3425f89dd09..0a3d8036dcc 100644 --- a/pom.xml +++ b/pom.xml @@ -106,7 +106,8 @@ true 1.3 4.5.1 - 2.9.3 + 2.9.3 + 1.9.17 smart-hadoop-3.3 smart-hadoop-client-3 1.5.3 @@ -504,6 +505,16 @@ caffeine ${caffeine.version} + + io.micrometer + micrometer-registry-prometheus + ${micrometer.version} + + + io.micrometer + micrometer-registry-jmx + ${micrometer.version} + org.apache.hadoop hadoop-common diff --git a/smart-agent/pom.xml b/smart-agent/pom.xml index 41692cf8337..b593cbdb760 100644 --- a/smart-agent/pom.xml +++ b/smart-agent/pom.xml @@ -34,6 +34,18 @@ 2.0.0-SNAPSHOT jar + + + + org.springframework.boot + spring-boot-dependencies + ${spring.boot.version} + pom + import + + + + org.smartdata @@ -41,6 +53,11 @@ 2.0.0-SNAPSHOT provided + + org.smartdata + smart-agent-web-server + 2.0.0-SNAPSHOT + com.typesafe.akka akka-testkit_2.11 diff --git a/smart-agent/src/main/java/org/smartdata/agent/SmartAgent.java b/smart-agent/src/main/java/org/smartdata/agent/SmartAgent.java index 8b657ca906f..fdd54aba1fe 100644 --- a/smart-agent/src/main/java/org/smartdata/agent/SmartAgent.java +++ b/smart-agent/src/main/java/org/smartdata/agent/SmartAgent.java @@ -34,17 +34,19 @@ import akka.util.Timeout; import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; +import io.micrometer.core.instrument.Tags; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.smartdata.AgentService; import org.smartdata.SmartConstants; +import org.smartdata.agent.http.SmartAgentHttpServer; import org.smartdata.conf.SmartConf; import org.smartdata.conf.SmartConfKeys; import org.smartdata.hdfs.HadoopUtil; +import org.smartdata.metrics.MetricsFactory; import org.smartdata.protocol.message.StatusMessage; import org.smartdata.protocol.message.StatusReporter; -import org.smartdata.server.engine.cmdlet.CmdletExecutor; import org.smartdata.server.engine.cmdlet.StatusReportTask; import org.smartdata.server.engine.cmdlet.agent.AgentCmdletService; import org.smartdata.server.engine.cmdlet.agent.AgentConstants; @@ -57,10 +59,10 @@ import org.smartdata.utils.SecurityUtil; import java.io.IOException; -import java.text.SimpleDateFormat; -import java.util.Date; +import java.util.Arrays; import java.util.Deque; import java.util.LinkedList; +import java.util.Optional; import java.util.UUID; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -73,34 +75,54 @@ public class SmartAgent implements StatusReporter { private static final String NAME = "SmartAgent"; private static final Logger LOG = LoggerFactory.getLogger(SmartAgent.class); + public static final Tags SMART_AGENT_BASE_TAGS = Tags.of("service", "smart-agent"); + + private final SmartAgentHttpServer httpServer; + private final SmartConf smartConfig; + private final String[] masters; + private final Config akkaConfig; private ActorSystem system; private ActorRef agentActor; - private CmdletExecutor cmdletExecutor; - public static void main(String[] args) throws IOException { - SmartAgent agent = new SmartAgent(); + public SmartAgent(SmartConf smartConfig) throws IOException { + this.masters = Optional.ofNullable(AgentUtils.getMasterAddress(smartConfig)) + .map(AgentUtils::getMasterActorPaths) + .orElseThrow(() -> new IOException("No master address found!")); + LOG.info("Agent masters: {}", Arrays.toString(masters)); - SmartConf conf = (SmartConf) new GenericOptionsParser( - new SmartConf(), args).getConfiguration(); + String agentAddress = AgentUtils.getAgentAddress(smartConfig); + LOG.info("Agent address: {}", agentAddress); + this.akkaConfig = AgentUtils.overrideRemoteAddress( + ConfigFactory.load(AgentConstants.AKKA_CONF_FILE), agentAddress); - String[] masters = AgentUtils.getMasterAddress(conf); - if (masters == null) { - throw new IOException("No master address found!"); - } - for (int i = 0; i < masters.length; i++) { - LOG.info("Agent master " + i + ": " + masters[i]); - } - String agentAddress = AgentUtils.getAgentAddress(conf); - LOG.info("Agent address: " + agentAddress); RegisterNewAgent.getInstance( "SSMAgent@" + agentAddress.replaceAll(":.*$", "")); + HadoopUtil.setSmartConfByHadoop(smartConfig); - HadoopUtil.setSmartConfByHadoop(conf); - agent.authentication(conf); + this.smartConfig = smartConfig; + this.httpServer = new SmartAgentHttpServer(smartConfig, + MetricsFactory.from(smartConfig, SMART_AGENT_BASE_TAGS)); + } - agent.start(AgentUtils.overrideRemoteAddress( - ConfigFactory.load(AgentConstants.AKKA_CONF_FILE), agentAddress), - AgentUtils.getMasterActorPaths(masters), conf); + public SmartAgent(SmartConf smartConf, Config akkaConfig, String[] masters) { + this.masters = masters; + this.akkaConfig = akkaConfig; + this.smartConfig = smartConf; + this.httpServer = new SmartAgentHttpServer(smartConf, + MetricsFactory.from(smartConf, SMART_AGENT_BASE_TAGS)); + } + + public static void main(String[] args) throws IOException { + SmartConf conf = (SmartConf) new GenericOptionsParser( + new SmartConf(), args).getConfiguration(); + SmartAgent smartAgent = SmartAgent.buildWith(conf); + smartAgent.start(); + } + + public static SmartAgent buildWith(SmartConf conf) throws IOException { + SmartAgent agent = new SmartAgent(conf); + agent.authentication(conf); + return agent; } //TODO: remove loadHadoopConf @@ -133,55 +155,54 @@ private void authentication(SmartConf conf) throws IOException { SecurityUtil.loginUsingKeytab(keytabFilename, principal); } - private static String getDateString() { - Date now = new Date(); - SimpleDateFormat df = new SimpleDateFormat("yyyyMMddHHmmss"); - return df.format(now); - } - - public void start(Config config, String[] masterPath, SmartConf conf) { - system = ActorSystem.apply(NAME, config); + public void start() { + system = ActorSystem.apply(NAME, akkaConfig); agentActor = system.actorOf(Props.create( - AgentActor.class, this, masterPath, conf), getAgentName()); + AgentActor.class, masters, smartConfig), getAgentName()); final Thread currentThread = Thread.currentThread(); - Runtime.getRuntime().addShutdownHook(new Thread() { - @Override - public void run() { - shutdown(); - try { - currentThread.join(); - } catch (InterruptedException e) { - // Ignore - } + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + shutdown(); + try { + currentThread.join(); + } catch (InterruptedException e) { + // Ignore } - }); - Services.init(new SmartAgentContext(conf, this)); + })); + Services.init(new SmartAgentContext(smartConfig, this)); Services.start(); AgentCmdletService agentCmdletService = (AgentCmdletService) Services.getService( SmartConstants.AGENT_CMDLET_SERVICE_NAME); - cmdletExecutor = agentCmdletService.getCmdletExecutor(); ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(); int reportPeriod = - conf.getInt(SmartConfKeys.SMART_STATUS_REPORT_PERIOD_KEY, + smartConfig.getInt(SmartConfKeys.SMART_STATUS_REPORT_PERIOD_KEY, SmartConfKeys.SMART_STATUS_REPORT_PERIOD_DEFAULT); StatusReportTask statusReportTask = - new StatusReportTask(this, cmdletExecutor, conf); + new StatusReportTask(this, agentCmdletService.getCmdletExecutor(), smartConfig); executorService.scheduleAtFixedRate( - statusReportTask, 1000, reportPeriod, TimeUnit.MILLISECONDS); + statusReportTask, 1000, reportPeriod, TimeUnit.MILLISECONDS); + + httpServer.start(); try { Await.result(system.whenTerminated(), Duration.Inf()); } catch (Exception e) { - LOG.error("Failure during actor system runtime.", e); - } + LOG.error("Failure during actor system runtime.", e); + } } public void shutdown() { Services.stop(); + + try { + httpServer.stop(); + } catch (Exception e) { + LOG.error("Error stopping agent http server", e); + } + if (system != null && !system.whenTerminated().isCompleted()) { LOG.info("Shutting down system {}", AgentUtils.getSystemAddres(system)); system.terminate(); @@ -198,7 +219,7 @@ public void report(StatusMessage status) { } private String getAgentName() { - return "agent-" + UUID.randomUUID().toString(); + return "agent-" + UUID.randomUUID(); } /** @@ -233,21 +254,19 @@ static class AgentActor extends UntypedActor { private static final FiniteDuration RETRY_INTERVAL = Duration.create(2, TimeUnit.SECONDS); + private final String[] masters; + private final SmartConf conf; + private final Deque unhandledMessages = new LinkedList<>(); private MasterToAgent.AgentId id; private ActorRef master; - private final SmartAgent agent; - private final String[] masters; - private SmartConf conf; - private Deque unhandledMessages = new LinkedList<>(); - public AgentActor(SmartAgent agent, String[] masters, SmartConf conf) { - this.agent = agent; + public AgentActor(String[] masters, SmartConf conf) { this.masters = masters; - this.conf = conf; + this.conf = conf; } @Override - public void onReceive(Object message) throws Exception { + public void onReceive(Object message) { unhandled(message); } @@ -272,8 +291,7 @@ public void preStart() { */ private Cancellable findMaster() { if (master != null) { - LOG.info("Before finding master, unwatch current master: " - + master.path().address()); + LOG.info("Before finding master, unwatch current master: {}", master.path().address()); this.context().unwatch(master); master = null; } @@ -289,7 +307,7 @@ public void run() { actorSelection.tell(new Identify(null), getSelf()); } } - }, new Shutdown(agent)); + }, new Shutdown()); } /** @@ -331,9 +349,9 @@ public void apply(Object message) throws Exception { // will be instantiated with this address. String rpcHost = master.path().address().host().get(); String rpcPort = conf - .get(SmartConfKeys.SMART_SERVER_RPC_ADDRESS_KEY, - SmartConfKeys.SMART_SERVER_RPC_ADDRESS_DEFAULT) - .split(":")[1]; + .get(SmartConfKeys.SMART_SERVER_RPC_ADDRESS_KEY, + SmartConfKeys.SMART_SERVER_RPC_ADDRESS_DEFAULT) + .split(":")[1]; conf.set(SmartConfKeys.SMART_SERVER_RPC_ADDRESS_KEY, rpcHost + ":" + rpcPort); @@ -341,7 +359,7 @@ public void apply(Object message) throws Exception { AgentUtils.repeatActionUntil(getContext().system(), Duration.Zero(), RETRY_INTERVAL, TIMEOUT, new SendMessage(master, RegisterNewAgent.getInstance()), - new Shutdown(agent)); + new Shutdown()); LOG.info("Registering to master {}", master); getContext().become(new WaitForRegisterAgent(registerAgent)); } @@ -372,7 +390,7 @@ public WaitForRegisterAgent(Cancellable registerAgent) { * message. */ @Override - public void apply(Object message) throws Exception { + public void apply(Object message) { if (message instanceof AgentRegistered) { AgentRegistered registered = (AgentRegistered) message; registerAgent.cancel(); @@ -391,7 +409,7 @@ public void apply(Object message) throws Exception { return; } LOG.warn("Received event: {}, details: {}", - associEvent.eventName(), associEvent.toString()); + associEvent.eventName(), associEvent); LOG.warn("Go back to the preceding context to find master.."); getContext().become(new WaitForFindMaster(findMaster())); } else { @@ -418,13 +436,12 @@ private void applyUnhandledMessage() { } LOG.info("Applying {} unhandled message(s)...", unhandledMessages.size()); - while (unhandledMessages.size() != 0) { + while (!unhandledMessages.isEmpty()) { Object message = unhandledMessages.pollFirst(); try { this.apply(message); } catch (Exception e) { - LOG.warn("Failed to handle message: " - + message.toString() + "Reason: " + e.getMessage()); + LOG.warn("Failed to handle message: {}Reason: {}", message.toString(), e.getMessage()); } } } @@ -441,7 +458,7 @@ private void applyUnhandledMessage() { * find new master if this event is received. */ @Override - public void apply(Object message) throws Exception { + public void apply(Object message) { if (message instanceof AgentService.Message) { try { Services.dispatch((AgentService.Message) message); @@ -467,11 +484,11 @@ public void apply(Object message) throws Exception { return; } LOG.warn("Received event: {}, details: {}", - associEvent.eventName(), associEvent.toString()); + associEvent.eventName(), associEvent); LOG.warn("Try to register to a new master..."); getContext().become(new WaitForFindMaster(findMaster())); } else { - LOG.warn("Unhandled message: " + message.toString()); + LOG.warn("Unhandled message: {}", message.toString()); } } } @@ -493,13 +510,6 @@ public void run() { } private class Shutdown implements Runnable { - - private SmartAgent agent; - - public Shutdown(SmartAgent agent) { - this.agent = agent; - } - /** * {@link SmartAgent#shutdown() shutdown} will be called before * the program exits. diff --git a/smart-agent/src/test/java/org.smartdata.agent/TestSmartAgent.java b/smart-agent/src/test/java/org.smartdata.agent/TestSmartAgent.java index 6b7dddcc1e4..07ad346c859 100644 --- a/smart-agent/src/test/java/org.smartdata.agent/TestSmartAgent.java +++ b/smart-agent/src/test/java/org.smartdata.agent/TestSmartAgent.java @@ -76,8 +76,8 @@ public AgentRunner(Config config, String[] masters) { @Override public void run() { - SmartAgent agent = new SmartAgent(); - agent.start(config, masters, new SmartConf()); + SmartAgent agent = new SmartAgent(new SmartConf(), config, masters); + agent.start(); } } @@ -87,6 +87,10 @@ public void run() { * waiting for actor system to shut down. */ static class NoExitSecurityManager extends SecurityManager { + @Override + public void checkPermission(Permission perm, Object context) { + } + @Override public void checkPermission(Permission perm) { } diff --git a/smart-common/pom.xml b/smart-common/pom.xml index 66888b5c799..3cede28ac8c 100644 --- a/smart-common/pom.xml +++ b/smart-common/pom.xml @@ -34,11 +34,6 @@ jar - - org.smartdata - smart-metrics - 2.0.0-SNAPSHOT - org.apache.hadoop hadoop-common diff --git a/smart-common/src/main/java/org/smartdata/SmartContext.java b/smart-common/src/main/java/org/smartdata/SmartContext.java index d365d479606..03731de89e4 100644 --- a/smart-common/src/main/java/org/smartdata/SmartContext.java +++ b/smart-common/src/main/java/org/smartdata/SmartContext.java @@ -17,12 +17,16 @@ */ package org.smartdata; +import lombok.Getter; +import lombok.Setter; import org.smartdata.conf.SmartConf; /** * SSM context for running an action. */ +@Setter +@Getter public class SmartContext { private SmartConf conf; @@ -35,11 +39,4 @@ public SmartContext(SmartConf conf) { this.conf = conf; } - public SmartConf getConf() { - return conf; - } - - public void setConf(SmartConf conf) { - this.conf = conf; - } } diff --git a/smart-common/src/main/java/org/smartdata/conf/SmartConfKeys.java b/smart-common/src/main/java/org/smartdata/conf/SmartConfKeys.java index 9e18e04f9eb..273ad44c068 100644 --- a/smart-common/src/main/java/org/smartdata/conf/SmartConfKeys.java +++ b/smart-common/src/main/java/org/smartdata/conf/SmartConfKeys.java @@ -17,6 +17,8 @@ */ package org.smartdata.conf; +import org.smartdata.metrics.impl.SmartServerAccessEventSource; + /** * This class contains the configure keys needed by SSM. */ @@ -54,6 +56,8 @@ public class SmartConfKeys { public static final String SMART_REST_SERVER_PORT_KEY = "smart.rest.server.port"; public static final int SMART_REST_SERVER_PORT_KEY_DEFAULT = 8081; + public static final String SMART_AGENT_HTTP_SERVER_PORT_KEY = "smart.agent.http.server.port"; + public static final int SMART_AGENT_HTTP_SERVER_PORT_DEFAULT = 8081; public static final String SMART_SECURITY_ENABLE = "smart.security.enable"; public static final String SMART_SERVER_KEYTAB_FILE_KEY = "smart.server.keytab.file"; public static final String SMART_SERVER_KERBEROS_PRINCIPAL_KEY = @@ -285,4 +289,29 @@ public class SmartConfKeys { public static final String SMART_ACTION_CLIENT_CACHE_TTL_KEY = "smart.action.client.cache.ttl"; public static final String SMART_ACTION_CLIENT_CACHE_TTL_DEFAULT = "10m"; + + public static final String ACCESS_EVENT_SOURCE_KEY = "smart.data.file.event.source"; + public static final String ACCESS_EVENT_SOURCE_DEFAULT = + SmartServerAccessEventSource.class.getName(); + + // Metrics + public static final String SMART_METRICS_ENABLED_KEY = + "smart.metrics.enabled"; + public static final boolean SMART_METRICS_ENABLED_DEFAULT = true; + + public static final String SMART_METRICS_JMX_ENABLED_KEY = + "smart.metrics.jmx.enabled"; + public static final boolean SMART_METRICS_JMX_ENABLED_DEFAULT = true; + + public static final String SMART_METRICS_JMX_DOMAIN_KEY = + "smart.metrics.jmx.domain"; + public static final String SMART_METRICS_JMX_DOMAIN_DEFAULT = "metrics"; + + public static final String SMART_METRICS_PROMETHEUS_ENABLED_KEY = + "smart.metrics.prometheus.enabled"; + public static final boolean SMART_METRICS_PROMETHEUS_ENABLED_DEFAULT = true; + + public static final String SMART_METRICS_DB_QUERIES_ENABLED_KEY = + "smart.metrics.db.queries.enabled"; + public static final boolean SMART_METRICS_DB_QUERIES_ENABLED_DEFAULT = false; } diff --git a/smart-metrics/src/main/java/org/smartdata/metrics/DataAccessEvent.java b/smart-common/src/main/java/org/smartdata/metrics/DataAccessEvent.java similarity index 100% rename from smart-metrics/src/main/java/org/smartdata/metrics/DataAccessEvent.java rename to smart-common/src/main/java/org/smartdata/metrics/DataAccessEvent.java diff --git a/smart-metrics/src/main/java/org/smartdata/metrics/FileAccessEvent.java b/smart-common/src/main/java/org/smartdata/metrics/FileAccessEvent.java similarity index 79% rename from smart-metrics/src/main/java/org/smartdata/metrics/FileAccessEvent.java rename to smart-common/src/main/java/org/smartdata/metrics/FileAccessEvent.java index 58e705ede22..ee801ab6380 100644 --- a/smart-metrics/src/main/java/org/smartdata/metrics/FileAccessEvent.java +++ b/smart-common/src/main/java/org/smartdata/metrics/FileAccessEvent.java @@ -17,13 +17,18 @@ */ package org.smartdata.metrics; +import lombok.Getter; +import lombok.Setter; + /** * A file access event. */ +@Getter public class FileAccessEvent implements DataAccessEvent { private final String path; private final String user; - private long timeStamp; + @Setter + private long timestamp; public FileAccessEvent(String path) { this(path, -1); @@ -33,9 +38,9 @@ public FileAccessEvent(String path, long timestamp) { this(path, timestamp, ""); } - public FileAccessEvent(String path, long timeStamp, String user) { + public FileAccessEvent(String path, long timestamp, String user) { this.path = path; - this.timeStamp = timeStamp; + this.timestamp = timestamp; this.user = user; } @@ -43,14 +48,6 @@ public FileAccessEvent(String path, String user) { this(path, -1, user); } - /** - * Get the accessed file path. - * @return file path - */ - public String getPath() { - return this.path; - } - // DFSClient has no info about the file id, except have another rpc call // to Namenode, SmartServer can get this value from Namespace, so not // provide id info here. @@ -62,13 +59,4 @@ public long getFileId() { public String getAccessedBy() { return this.user; } - - @Override - public long getTimestamp() { - return this.timeStamp; - } - - public void setTimeStamp(long timeStamp) { - this.timeStamp = timeStamp; - } } diff --git a/smart-metrics/src/main/java/org/smartdata/metrics/FileAccessEventCollector.java b/smart-common/src/main/java/org/smartdata/metrics/FileAccessEventCollector.java similarity index 100% rename from smart-metrics/src/main/java/org/smartdata/metrics/FileAccessEventCollector.java rename to smart-common/src/main/java/org/smartdata/metrics/FileAccessEventCollector.java diff --git a/smart-metrics/src/main/java/org/smartdata/metrics/FileAccessEventSource.java b/smart-common/src/main/java/org/smartdata/metrics/FileAccessEventSource.java similarity index 100% rename from smart-metrics/src/main/java/org/smartdata/metrics/FileAccessEventSource.java rename to smart-common/src/main/java/org/smartdata/metrics/FileAccessEventSource.java diff --git a/smart-metrics/src/main/java/org/smartdata/metrics/impl/MetricsFactory.java b/smart-common/src/main/java/org/smartdata/metrics/impl/FileAccessMetricsFactory.java similarity index 71% rename from smart-metrics/src/main/java/org/smartdata/metrics/impl/MetricsFactory.java rename to smart-common/src/main/java/org/smartdata/metrics/impl/FileAccessMetricsFactory.java index bc383b70e38..08a1d32d339 100644 --- a/smart-metrics/src/main/java/org/smartdata/metrics/impl/MetricsFactory.java +++ b/smart-common/src/main/java/org/smartdata/metrics/impl/FileAccessMetricsFactory.java @@ -17,7 +17,6 @@ */ package org.smartdata.metrics.impl; -import org.apache.hadoop.conf.Configuration; import org.smartdata.metrics.FileAccessEventSource; import java.io.IOException; @@ -25,19 +24,14 @@ /** * A factory used to create FileAccessEventSource according to the configuration. */ -public class MetricsFactory { - private static final String ACCESS_EVENT_SOURCE = "smart.data.file.event.source"; - private static final String DEFAULT_ACCESS_EVENT_SOURCE = - SmartServerAccessEventSource.class.getName(); +public class FileAccessMetricsFactory { - public static FileAccessEventSource createAccessEventSource(Configuration conf) + public static FileAccessEventSource createAccessEventSource(String source) throws IOException { - String source = conf.get(ACCESS_EVENT_SOURCE, DEFAULT_ACCESS_EVENT_SOURCE); try { - Class clazz = Class.forName(source); + Class clazz = Class.forName(source); return (FileAccessEventSource) clazz.newInstance(); } catch (ClassNotFoundException | IllegalAccessException | InstantiationException e) { - e.printStackTrace(); throw new IOException(e); } } diff --git a/smart-metrics/src/main/java/org/smartdata/metrics/impl/SmartServerAccessEventCollector.java b/smart-common/src/main/java/org/smartdata/metrics/impl/SmartServerAccessEventCollector.java similarity index 100% rename from smart-metrics/src/main/java/org/smartdata/metrics/impl/SmartServerAccessEventCollector.java rename to smart-common/src/main/java/org/smartdata/metrics/impl/SmartServerAccessEventCollector.java diff --git a/smart-metrics/src/main/java/org/smartdata/metrics/impl/SmartServerAccessEventSource.java b/smart-common/src/main/java/org/smartdata/metrics/impl/SmartServerAccessEventSource.java similarity index 64% rename from smart-metrics/src/main/java/org/smartdata/metrics/impl/SmartServerAccessEventSource.java rename to smart-common/src/main/java/org/smartdata/metrics/impl/SmartServerAccessEventSource.java index 2ecb01725ab..e9eec474d9a 100644 --- a/smart-metrics/src/main/java/org/smartdata/metrics/impl/SmartServerAccessEventSource.java +++ b/smart-common/src/main/java/org/smartdata/metrics/impl/SmartServerAccessEventSource.java @@ -23,26 +23,20 @@ import org.smartdata.metrics.FileAccessEventCollector; import org.smartdata.metrics.FileAccessEventSource; -import java.util.Timer; -import java.util.TimerTask; import java.util.concurrent.LinkedBlockingQueue; /** * The default AccessEventSource for SmartServer. */ public class SmartServerAccessEventSource implements FileAccessEventSource { - static final Logger LOG = LoggerFactory.getLogger(SmartServerAccessEventSource.class); + private static final Logger LOG = LoggerFactory.getLogger(SmartServerAccessEventSource.class); - private static final long DEFAULT_INTERVAL = 1000; //5 seconds private final SmartServerAccessEventCollector collector; - private LinkedBlockingQueue eventQueue; - private Timer timer; + private final LinkedBlockingQueue eventQueue; public SmartServerAccessEventSource() { - this.timer = new Timer(); this.eventQueue = new LinkedBlockingQueue<>(); this.collector = new SmartServerAccessEventCollector(eventQueue); - this.timer.schedule(new ProgressInsertTask(eventQueue), DEFAULT_INTERVAL, DEFAULT_INTERVAL); } @Override @@ -54,7 +48,7 @@ public FileAccessEventCollector getCollector() { public void insertEventFromSmartClient(FileAccessEvent event) { try { this.eventQueue.put(event); - LOG.trace("Access:" + event.getPath()); + LOG.trace("Access: {}", event.getPath()); } catch (InterruptedException e) { LOG.error("Event queue enqueue path={} error", event.getPath(), e); } @@ -62,24 +56,6 @@ public void insertEventFromSmartClient(FileAccessEvent event) { @Override public void close() { - this.timer.cancel(); - } - - private static class ProgressInsertTask extends TimerTask { - private final LinkedBlockingQueue outerQueue; - public ProgressInsertTask(LinkedBlockingQueue outerQueue) { - this.outerQueue = outerQueue; - } - - @Override - public void run() { - try { - //Todo: do not use HDFSFileAccessEvent - this.outerQueue.put(new FileAccessEvent("", System.currentTimeMillis())); - } catch (InterruptedException e) { - LOG.error("Outer queue enqueue error", e); - } - } } } diff --git a/smart-common/src/main/java/org/smartdata/security/NoOpSmartPrincipalManager.java b/smart-common/src/main/java/org/smartdata/security/NoOpSmartPrincipalManager.java new file mode 100644 index 00000000000..4859d05349b --- /dev/null +++ b/smart-common/src/main/java/org/smartdata/security/NoOpSmartPrincipalManager.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.security; + +public class NoOpSmartPrincipalManager implements SmartPrincipalManager { + @Override + public SmartPrincipal getCurrentPrincipal() { + return null; + } + + @Override + public void setCurrentPrincipal(SmartPrincipal principal) { + + } + + @Override + public void unsetCurrentPrincipal() { + + } +} diff --git a/smart-common/src/main/java/org/smartdata/security/SmartPrincipalManager.java b/smart-common/src/main/java/org/smartdata/security/SmartPrincipalManager.java index ccfeffad612..c6b12a754c0 100644 --- a/smart-common/src/main/java/org/smartdata/security/SmartPrincipalManager.java +++ b/smart-common/src/main/java/org/smartdata/security/SmartPrincipalManager.java @@ -23,4 +23,8 @@ public interface SmartPrincipalManager { void setCurrentPrincipal(SmartPrincipal principal); void unsetCurrentPrincipal(); + + static SmartPrincipalManager noOp() { + return new NoOpSmartPrincipalManager(); + } } diff --git a/smart-dist/pom.xml b/smart-dist/pom.xml index 0ea97914642..225ba5b0202 100644 --- a/smart-dist/pom.xml +++ b/smart-dist/pom.xml @@ -74,11 +74,6 @@ ${smart.hadoop.client.artifact} 2.0.0-SNAPSHOT - - org.smartdata - smart-metrics - 2.0.0-SNAPSHOT - org.smartdata smart-agent diff --git a/smart-engine/src/main/java/org/smartdata/server/SmartEngine.java b/smart-engine/src/main/java/org/smartdata/server/SmartEngine.java index 12217da099e..5baa36de470 100644 --- a/smart-engine/src/main/java/org/smartdata/server/SmartEngine.java +++ b/smart-engine/src/main/java/org/smartdata/server/SmartEngine.java @@ -40,6 +40,7 @@ public class SmartEngine extends AbstractService { public static final Logger LOG = LoggerFactory.getLogger(SmartEngine.class); private final SmartConf conf; + @Getter private final ServerContext serverContext; @Getter private StatesManager statesManager; @@ -114,8 +115,7 @@ private void stopEngineService(AbstractService service) { service.stop(); } } catch (IOException e) { - LOG.error("Error while stopping " - + service.getClass().getCanonicalName(), e); + LOG.error("Error while stopping {}", service.getClass().getCanonicalName(), e); } } diff --git a/smart-engine/src/main/java/org/smartdata/server/engine/CmdletManager.java b/smart-engine/src/main/java/org/smartdata/server/engine/CmdletManager.java index 3715f566bd2..d592acc34a9 100644 --- a/smart-engine/src/main/java/org/smartdata/server/engine/CmdletManager.java +++ b/smart-engine/src/main/java/org/smartdata/server/engine/CmdletManager.java @@ -142,7 +142,7 @@ public class CmdletManager extends AbstractService private final AuditService auditService; private final SmartPrincipalManager smartPrincipalManager; private final PathChecker pathChecker; - private List schedulerServices; + private final List schedulerServices; private CmdletDispatcher dispatcher; public CmdletManager( @@ -152,7 +152,8 @@ public CmdletManager( super(context); this.metaStore = context.getMetaStore(); - this.executorService = Executors.newScheduledThreadPool(4); + this.executorService = context.getMetricsFactory().wrap( + Executors.newScheduledThreadPool(4), "cmdletSchedulerExecutor"); this.runningCmdlets = new ArrayList<>(); this.pendingCmdlets = new LinkedList<>(); this.schedulingCmdlets = new LinkedList<>(); @@ -171,8 +172,8 @@ public CmdletManager( CacheScheduler.class ), context, metaStore); this.ruleCmdletTracker = new RuleCmdletTracker(); - this.dispatcher = new CmdletDispatcher(context, this, scheduledCmdlets, - idToLaunchCmdlets, runningCmdlets, schedulers); + this.dispatcher = new CmdletDispatcher(context, this, + scheduledCmdlets, idToLaunchCmdlets, runningCmdlets, schedulers); this.pathChecker = new PathChecker(context.getConf()); this.maxNumPendingCmdlets = context.getConf() .getInt(SmartConfKeys.SMART_CMDLET_MAX_NUM_PENDING_KEY, @@ -184,7 +185,7 @@ public CmdletManager( this.inMemoryRegistry = new InMemoryRegistry(context, ruleCmdletTracker, executorService); CmdletManagerContext cmdletManagerContext = new CmdletManagerContext( - getContext().getConf(), metaStore, inMemoryRegistry, schedulers); + context.getConf(), metaStore, context.getMetricsFactory(), inMemoryRegistry, schedulers); this.detectTimeoutActionsTask = new DetectTimeoutActionsTask(cmdletManagerContext, this, idToLaunchCmdlets.keySet()); this.actionInfoHandler = new ActionInfoHandler(cmdletManagerContext); @@ -696,7 +697,7 @@ public void deleteCmdletByRule(long rid) throws IOException { public void updateStatus(StatusMessage status) { if (LOG.isDebugEnabled()) { - LOG.debug("Got status update: " + status); + LOG.debug("Got status update: {}", status); } try { if (status instanceof CmdletStatusUpdate) { @@ -706,7 +707,7 @@ public void updateStatus(StatusMessage status) { onStatusReport((StatusReport) status); } } catch (IOException e) { - LOG.error(String.format("Update status %s failed with %s", status, e)); + LOG.error("Update status {} failed with {}", status, e); } catch (ActionException e) { LOG.error("Action Status error", e); } diff --git a/smart-engine/src/main/java/org/smartdata/server/engine/ServerContext.java b/smart-engine/src/main/java/org/smartdata/server/engine/ServerContext.java index 3aea431b22b..303caccde34 100644 --- a/smart-engine/src/main/java/org/smartdata/server/engine/ServerContext.java +++ b/smart-engine/src/main/java/org/smartdata/server/engine/ServerContext.java @@ -17,25 +17,25 @@ */ package org.smartdata.server.engine; +import lombok.Getter; import org.smartdata.SmartContext; import org.smartdata.conf.SmartConf; import org.smartdata.metastore.MetaStore; +import org.smartdata.metrics.MetricsFactory; +@Getter public class ServerContext extends SmartContext { private final MetaStore metaStore; + private final MetricsFactory metricsFactory; - - public ServerContext(MetaStore metaStore) { - this.metaStore = metaStore; + public ServerContext(SmartConf conf, MetaStore metaStore) { + this(conf, metaStore, MetricsFactory.noOp()); } - public ServerContext(SmartConf conf, MetaStore metaStore) { + public ServerContext(SmartConf conf, MetaStore metaStore, MetricsFactory metricsFactory) { super(conf); this.metaStore = metaStore; - } - - public MetaStore getMetaStore() { - return metaStore; + this.metricsFactory = metricsFactory; } } diff --git a/smart-engine/src/main/java/org/smartdata/server/engine/StatesManager.java b/smart-engine/src/main/java/org/smartdata/server/engine/StatesManager.java index 66ae9726cff..7ba25026392 100644 --- a/smart-engine/src/main/java/org/smartdata/server/engine/StatesManager.java +++ b/smart-engine/src/main/java/org/smartdata/server/engine/StatesManager.java @@ -35,7 +35,7 @@ import org.smartdata.metastore.transaction.TransactionRunner; import org.smartdata.metrics.FileAccessEvent; import org.smartdata.metrics.FileAccessEventSource; -import org.smartdata.metrics.impl.MetricsFactory; +import org.smartdata.metrics.impl.FileAccessMetricsFactory; import org.smartdata.model.PathChecker; import org.smartdata.server.engine.data.AccessEventFetcher; @@ -45,6 +45,8 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import static org.smartdata.conf.SmartConfKeys.ACCESS_EVENT_SOURCE_DEFAULT; +import static org.smartdata.conf.SmartConfKeys.ACCESS_EVENT_SOURCE_KEY; import static org.springframework.transaction.annotation.Isolation.SERIALIZABLE; /** @@ -86,7 +88,10 @@ public void init() throws IOException { transactionRunner, serverContext.getMetaStore().accessCountEventDao(), serverContext.getMetaStore().cacheFileDao()); - this.fileAccessEventSource = MetricsFactory.createAccessEventSource(serverContext.getConf()); + + String accessEventSource = serverContext.getConf().get( + ACCESS_EVENT_SOURCE_KEY, ACCESS_EVENT_SOURCE_DEFAULT); + this.fileAccessEventSource = FileAccessMetricsFactory.createAccessEventSource(accessEventSource); AccessCountFailoverFactory accessCountFailoverFactory = new AccessCountFailoverFactory(serverContext.getConf()); DbAccessEventAggregator accessEventAggregator = new DbAccessEventAggregator( @@ -97,7 +102,8 @@ public void init() throws IOException { serverContext.getConf(), accessEventAggregator, executorService, - fileAccessEventSource.getCollector()); + fileAccessEventSource.getCollector(), + serverContext.getMetricsFactory()); this.pathChecker = new PathChecker(serverContext.getConf()); this.cachedFilesManager = new CachedFilesManager(serverContext.getMetaStore().cacheFileDao()); @@ -181,7 +187,7 @@ public void reportFileAccessEvent(FileAccessEvent event) { LOG.debug("Path {} is not in the whitelist. Report file access event failed.", path); return; } - event.setTimeStamp(System.currentTimeMillis()); + event.setTimestamp(System.currentTimeMillis()); this.fileAccessEventSource.insertEventFromSmartClient(event); } diff --git a/smart-engine/src/main/java/org/smartdata/server/engine/cmdlet/CmdletDispatcher.java b/smart-engine/src/main/java/org/smartdata/server/engine/cmdlet/CmdletDispatcher.java index 5feae6c9802..eb9287e101f 100644 --- a/smart-engine/src/main/java/org/smartdata/server/engine/cmdlet/CmdletDispatcher.java +++ b/smart-engine/src/main/java/org/smartdata/server/engine/cmdlet/CmdletDispatcher.java @@ -20,7 +20,6 @@ import com.google.common.collect.ListMultimap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.smartdata.SmartContext; import org.smartdata.conf.SmartConf; import org.smartdata.conf.SmartConfKeys; import org.smartdata.model.CmdletState; @@ -35,6 +34,7 @@ import org.smartdata.server.cluster.NodeCmdletMetrics; import org.smartdata.server.engine.ActiveServerInfo; import org.smartdata.server.engine.CmdletManager; +import org.smartdata.server.engine.ServerContext; import org.smartdata.server.engine.message.NodeMessage; import java.io.IOException; @@ -86,7 +86,7 @@ public class CmdletDispatcher implements ClusterNodeMetricsProvider { private final SmartConf conf; - public CmdletDispatcher(SmartContext smartContext, CmdletManager cmdletManager, + public CmdletDispatcher(ServerContext smartContext, CmdletManager cmdletManager, Queue scheduledCmdlets, Map idToLaunchCmdlet, List runningCmdlets, ListMultimap schedulers) { this.conf = smartContext.getConf(); @@ -135,7 +135,8 @@ public CmdletDispatcher(SmartContext smartContext, CmdletManager cmdletManager, for (int i = 0; i < numDisp; i++) { dispatchTasks[i] = new DispatchTask(this, i); } - this.schExecService = Executors.newScheduledThreadPool(numDisp + 1); + this.schExecService = smartContext.getMetricsFactory().wrap( + Executors.newScheduledThreadPool(numDisp + 1), "cmdletDispatcherExecutor"); this.outputDispMetricsInterval = conf.getInt( SmartConfKeys.SMART_CMDLET_DISPATCHER_LOG_DISP_METRICS_INTERVAL_KEY, SmartConfKeys.SMART_CMDLET_DISPATCHER_LOG_DISP_METRICS_INTERVAL_DEFAULT); diff --git a/smart-engine/src/main/java/org/smartdata/server/engine/cmdlet/CmdletManagerContext.java b/smart-engine/src/main/java/org/smartdata/server/engine/cmdlet/CmdletManagerContext.java index 46ee347918a..9594d285b99 100644 --- a/smart-engine/src/main/java/org/smartdata/server/engine/cmdlet/CmdletManagerContext.java +++ b/smart-engine/src/main/java/org/smartdata/server/engine/cmdlet/CmdletManagerContext.java @@ -18,31 +18,31 @@ package org.smartdata.server.engine.cmdlet; import com.google.common.collect.ListMultimap; +import lombok.Getter; import org.smartdata.conf.SmartConf; import org.smartdata.metastore.MetaStore; +import org.smartdata.metrics.MetricsFactory; import org.smartdata.model.action.ActionScheduler; import org.smartdata.server.engine.ServerContext; import java.util.List; public class CmdletManagerContext extends ServerContext { + @Getter private final InMemoryRegistry inMemoryRegistry; private final ListMultimap schedulers; public CmdletManagerContext( SmartConf conf, MetaStore metaStore, + MetricsFactory metricsFactory, InMemoryRegistry inMemoryRegistry, ListMultimap schedulers) { - super(conf, metaStore); + super(conf, metaStore, metricsFactory); this.inMemoryRegistry = inMemoryRegistry; this.schedulers = schedulers; } - public InMemoryRegistry getInMemoryRegistry() { - return inMemoryRegistry; - } - public List getSchedulers(String action) { return schedulers.get(action); } diff --git a/smart-engine/src/main/java/org/smartdata/server/engine/data/AccessEventFetcher.java b/smart-engine/src/main/java/org/smartdata/server/engine/data/AccessEventFetcher.java index e50a54fa293..108f438861f 100644 --- a/smart-engine/src/main/java/org/smartdata/server/engine/data/AccessEventFetcher.java +++ b/smart-engine/src/main/java/org/smartdata/server/engine/data/AccessEventFetcher.java @@ -17,12 +17,14 @@ */ package org.smartdata.server.engine.data; +import io.micrometer.core.instrument.Counter; import org.apache.hadoop.conf.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.smartdata.metastore.accesscount.AccessEventAggregator; import org.smartdata.metrics.FileAccessEvent; import org.smartdata.metrics.FileAccessEventCollector; +import org.smartdata.metrics.MetricsFactory; import java.io.IOException; import java.util.List; @@ -45,12 +47,14 @@ public AccessEventFetcher( Configuration conf, AccessEventAggregator accessEventAggregator, ScheduledExecutorService service, - FileAccessEventCollector collector) { + FileAccessEventCollector collector, + MetricsFactory metricsFactory) { this.fetchInterval = conf.getLong( SMART_ACCESS_EVENT_FETCH_INTERVAL_MS_KEY, SMART_ACCESS_EVENT_FETCH_INTERVAL_MS_DEFAULT ); - this.fetchTask = new FetchTask(accessEventAggregator, collector); + Counter eventsCounter = metricsFactory.counter("access-events-count"); + this.fetchTask = new FetchTask(accessEventAggregator, collector, eventsCounter); this.scheduledExecutorService = service; } @@ -68,11 +72,13 @@ public void stop() { private static class FetchTask implements Runnable { private final AccessEventAggregator accessEventAggregator; private final FileAccessEventCollector collector; + private final Counter eventsCounter; public FetchTask(AccessEventAggregator accessEventAggregator, - FileAccessEventCollector collector) { + FileAccessEventCollector collector, Counter eventsCounter) { this.accessEventAggregator = accessEventAggregator; this.collector = collector; + this.eventsCounter = eventsCounter; } @Override @@ -80,6 +86,7 @@ public void run() { try { List events = this.collector.collect(); if (!events.isEmpty()) { + eventsCounter.increment(events.size()); accessEventAggregator.aggregate(events); } } catch (IOException e) { diff --git a/smart-hadoop-support/smart-hadoop-client-3/pom.xml b/smart-hadoop-support/smart-hadoop-client-3/pom.xml index 1d07526f94d..ffb90562776 100644 --- a/smart-hadoop-support/smart-hadoop-client-3/pom.xml +++ b/smart-hadoop-support/smart-hadoop-client-3/pom.xml @@ -40,12 +40,6 @@ 2.0.0-SNAPSHOT provided - - org.smartdata - smart-metrics - 2.0.0-SNAPSHOT - provided - org.apache.hadoop hadoop-hdfs diff --git a/smart-hadoop-support/smart-hadoop-client-3/src/main/java/org/smartdata/hdfs/client/SmartDFSClient.java b/smart-hadoop-support/smart-hadoop-client-3/src/main/java/org/smartdata/hdfs/client/SmartDFSClient.java index 0a9bbd6a2b7..8fc197f03f4 100644 --- a/smart-hadoop-support/smart-hadoop-client-3/src/main/java/org/smartdata/hdfs/client/SmartDFSClient.java +++ b/smart-hadoop-support/smart-hadoop-client-3/src/main/java/org/smartdata/hdfs/client/SmartDFSClient.java @@ -42,7 +42,6 @@ import org.slf4j.LoggerFactory; import org.smartdata.SmartConstants; import org.smartdata.client.SmartClient; -import org.smartdata.hdfs.CompatibilityHelperLoader; import org.smartdata.metrics.FileAccessEvent; import org.smartdata.model.CompactFileState; import org.smartdata.model.CompressionFileState; diff --git a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/NNMetricsAccessEventCollector.java b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/NNMetricsAccessEventCollector.java index cb4ac467853..165738dcfcb 100644 --- a/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/NNMetricsAccessEventCollector.java +++ b/smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/NNMetricsAccessEventCollector.java @@ -21,7 +21,6 @@ import org.apache.commons.configuration2.PropertiesConfiguration; import org.apache.commons.configuration2.SubsetConfiguration; import org.apache.commons.configuration2.builder.fluent.Configurations; -import org.apache.commons.configuration2.builder.fluent.Parameters; import org.apache.commons.configuration2.ex.ConfigurationException; import org.apache.commons.lang3.time.FastDateFormat; import org.apache.hadoop.conf.Configuration; diff --git a/smart-integration/pom.xml b/smart-integration/pom.xml index 482e5fd07ad..ea22426d0b1 100644 --- a/smart-integration/pom.xml +++ b/smart-integration/pom.xml @@ -253,7 +253,7 @@ generate - ${project.basedir}/../smart-web-server/src/main/resources/api/ssm-api.yaml + ${project.basedir}/../smart-web-server/smart-master-web-server/src/main/resources/api/ssm-api.yaml java java8 diff --git a/smart-integration/src/test/java/org/smartdata/integration/TestRuleRestApi.java b/smart-integration/src/test/java/org/smartdata/integration/TestRuleRestApi.java index 85d4b19fd6c..36702e72641 100644 --- a/smart-integration/src/test/java/org/smartdata/integration/TestRuleRestApi.java +++ b/smart-integration/src/test/java/org/smartdata/integration/TestRuleRestApi.java @@ -26,8 +26,8 @@ import org.smartdata.client.generated.model.RuleStateDto; import org.smartdata.client.generated.model.RulesDto; import org.smartdata.client.generated.model.RulesInfoDto; +import org.smartdata.http.error.SsmErrorCode; import org.smartdata.integration.api.RulesApiWrapper; -import org.smartdata.server.error.SsmErrorCode; import java.time.Duration; diff --git a/smart-integration/src/test/java/org/smartdata/integration/TestSsl.java b/smart-integration/src/test/java/org/smartdata/integration/TestSsl.java index 6dbbc8488eb..84792c629ea 100644 --- a/smart-integration/src/test/java/org/smartdata/integration/TestSsl.java +++ b/smart-integration/src/test/java/org/smartdata/integration/TestSsl.java @@ -40,11 +40,11 @@ import static io.restassured.config.RestAssuredConfig.config; import static org.junit.Assert.assertEquals; import static org.smartdata.client.generated.invoker.JacksonObjectMapper.jackson; -import static org.smartdata.server.config.ConfigKeys.SSL_ENABLED; -import static org.smartdata.server.config.ConfigKeys.SSL_KEYSTORE_PASSWORD; -import static org.smartdata.server.config.ConfigKeys.SSL_KEYSTORE_PATH; -import static org.smartdata.server.config.ConfigKeys.SSL_KEY_ALIAS; -import static org.smartdata.server.config.ConfigKeys.SSL_KEY_PASSWORD; +import static org.smartdata.http.config.ConfigKeys.SSL_ENABLED; +import static org.smartdata.http.config.ConfigKeys.SSL_KEYSTORE_PASSWORD; +import static org.smartdata.http.config.ConfigKeys.SSL_KEYSTORE_PATH; +import static org.smartdata.http.config.ConfigKeys.SSL_KEY_ALIAS; +import static org.smartdata.http.config.ConfigKeys.SSL_KEY_PASSWORD; @RunWith(Parameterized.class) public class TestSsl extends IntegrationTestBase { diff --git a/smart-integration/src/test/java/org/smartdata/integration/auth/TestWebServerAuth.java b/smart-integration/src/test/java/org/smartdata/integration/auth/TestWebServerAuth.java index 79aef3df93f..ff8435444cf 100644 --- a/smart-integration/src/test/java/org/smartdata/integration/auth/TestWebServerAuth.java +++ b/smart-integration/src/test/java/org/smartdata/integration/auth/TestWebServerAuth.java @@ -41,7 +41,7 @@ import static org.junit.Assert.assertTrue; import static org.smartdata.client.generated.invoker.ApiClient.BASE_URI; import static org.smartdata.client.generated.invoker.JacksonObjectMapper.jackson; -import static org.smartdata.server.config.ConfigKeys.WEB_SECURITY_ENABLED; +import static org.smartdata.http.config.ConfigKeys.WEB_SECURITY_ENABLED; @RunWith(Parameterized.class) public abstract class TestWebServerAuth extends IntegrationTestBase { diff --git a/smart-integration/src/test/java/org/smartdata/integration/auth/TestWebServerLdapAuth.java b/smart-integration/src/test/java/org/smartdata/integration/auth/TestWebServerLdapAuth.java index 022a05dbbcf..36919b7cf20 100644 --- a/smart-integration/src/test/java/org/smartdata/integration/auth/TestWebServerLdapAuth.java +++ b/smart-integration/src/test/java/org/smartdata/integration/auth/TestWebServerLdapAuth.java @@ -22,28 +22,28 @@ import org.junit.BeforeClass; import org.junit.runners.Parameterized.Parameters; import org.smartdata.conf.SmartConf; -import org.smartdata.server.config.PasswordEncoderFactory.EncoderType; -import org.smartdata.server.config.ldap.search.LdapSearchScope; - +import org.smartdata.http.config.PasswordEncoderFactory.EncoderType; +import org.smartdata.http.config.ldap.search.LdapSearchScope; + +import static org.smartdata.http.config.ConfigKeys.LDAP_AUTH_ENABLED; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_AUTH_TYPE; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_BIND_PASSWORD; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_BIND_USER_DN; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_CUSTOM_SEARCH; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_GROUP_MEMBER_ATTR; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_GROUP_NAME_ATTR; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_GROUP_OBJECT_DEFAULT; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_PASSWORD_ENCODER; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_SEARCH_ADDITIONAL_FILTER; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_SEARCH_BASE; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_URL; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_GROUPS; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_MEMBER_ATTR; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_SEARCH_BASE; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_SEARCH_SCOPE; import static org.smartdata.integration.auth.TestWebServerAuth.TestParams.ExpectedResult.FAIL; import static org.smartdata.integration.auth.TestWebServerLdapAuth.AuthType.BIND; import static org.smartdata.integration.auth.TestWebServerLdapAuth.AuthType.PASSWORD_COMPARE; -import static org.smartdata.server.config.ConfigKeys.LDAP_AUTH_ENABLED; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_AUTH_TYPE; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_BIND_PASSWORD; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_BIND_USER_DN; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_CUSTOM_SEARCH; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_GROUP_MEMBER_ATTR; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_GROUP_NAME_ATTR; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_GROUP_OBJECT_DEFAULT; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_PASSWORD_ENCODER; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_SEARCH_ADDITIONAL_FILTER; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_SEARCH_BASE; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_URL; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_GROUPS; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_MEMBER_ATTR; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_SEARCH_BASE; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_SEARCH_SCOPE; public class TestWebServerLdapAuth extends TestWebServerAuth { diff --git a/smart-integration/src/test/java/org/smartdata/integration/auth/TestWebServerPredefinedUsersAuth.java b/smart-integration/src/test/java/org/smartdata/integration/auth/TestWebServerPredefinedUsersAuth.java index 03f6d8602f3..c8f2d6f0e03 100644 --- a/smart-integration/src/test/java/org/smartdata/integration/auth/TestWebServerPredefinedUsersAuth.java +++ b/smart-integration/src/test/java/org/smartdata/integration/auth/TestWebServerPredefinedUsersAuth.java @@ -20,10 +20,10 @@ import org.junit.runners.Parameterized.Parameters; import org.smartdata.conf.SmartConf; +import static org.smartdata.http.config.ConfigKeys.PREDEFINED_BASIC_AUTH_ENABLED; +import static org.smartdata.http.config.ConfigKeys.PREDEFINED_USERS; +import static org.smartdata.http.config.ConfigKeys.PREDEFINED_USERS_PASSWORD_ENCODER; import static org.smartdata.integration.auth.TestWebServerAuth.TestParams.ExpectedResult.FAIL; -import static org.smartdata.server.config.ConfigKeys.PREDEFINED_BASIC_AUTH_ENABLED; -import static org.smartdata.server.config.ConfigKeys.PREDEFINED_USERS; -import static org.smartdata.server.config.ConfigKeys.PREDEFINED_USERS_PASSWORD_ENCODER; public class TestWebServerPredefinedUsersAuth extends TestWebServerAuth { diff --git a/smart-metastore/pom.xml b/smart-metastore/pom.xml index 32c5af7e4df..5306fc6fcf5 100644 --- a/smart-metastore/pom.xml +++ b/smart-metastore/pom.xml @@ -45,6 +45,11 @@ smart-common 2.0.0-SNAPSHOT + + org.smartdata + smart-metrics + 2.0.0-SNAPSHOT + com.alibaba druid diff --git a/smart-metastore/src/main/java/org/smartdata/metastore/DBPool.java b/smart-metastore/src/main/java/org/smartdata/metastore/DBPool.java index 289d3e37107..ce6279f5e58 100644 --- a/smart-metastore/src/main/java/org/smartdata/metastore/DBPool.java +++ b/smart-metastore/src/main/java/org/smartdata/metastore/DBPool.java @@ -17,6 +17,8 @@ */ package org.smartdata.metastore; +import org.smartdata.metrics.MetricsFactory; + import javax.sql.DataSource; import java.sql.Connection; @@ -30,4 +32,8 @@ public interface DBPool { void closeConnection(Connection conn) throws SQLException; void close(); + + default void bindMetrics(MetricsFactory metricFactory) { + // do nothing + } } diff --git a/smart-metastore/src/main/java/org/smartdata/metastore/DruidPool.java b/smart-metastore/src/main/java/org/smartdata/metastore/DruidPool.java index 5111bd4a1d2..c9f9ad05a9f 100644 --- a/smart-metastore/src/main/java/org/smartdata/metastore/DruidPool.java +++ b/smart-metastore/src/main/java/org/smartdata/metastore/DruidPool.java @@ -19,6 +19,8 @@ import com.alibaba.druid.pool.DruidDataSource; import com.alibaba.druid.pool.DruidDataSourceFactory; +import org.smartdata.conf.SmartConf; +import org.smartdata.metrics.MetricsFactory; import javax.sql.DataSource; @@ -28,11 +30,13 @@ public class DruidPool implements DBPool { private final DruidDataSource ds; + private final DruidPoolMetricsBinder metricsBinder; - public DruidPool(Properties properties) throws MetaStoreException { + public DruidPool(SmartConf conf, Properties properties) throws MetaStoreException { try { ds = (DruidDataSource) DruidDataSourceFactory.createDataSource(properties); + metricsBinder = DruidPoolMetricsBinder.build(conf, ds); } catch (Exception e) { throw new MetaStoreException(e); } @@ -50,6 +54,11 @@ public void closeConnection(Connection conn) throws SQLException { conn.close(); } + @Override + public void bindMetrics(MetricsFactory metricFactory) { + metricsBinder.bindTo(metricFactory.getMeterRegistry()); + } + public void close() { ds.close(); } diff --git a/smart-metastore/src/main/java/org/smartdata/metastore/DruidPoolMetricsBinder.java b/smart-metastore/src/main/java/org/smartdata/metastore/DruidPoolMetricsBinder.java new file mode 100644 index 00000000000..ee3ed7b826b --- /dev/null +++ b/smart-metastore/src/main/java/org/smartdata/metastore/DruidPoolMetricsBinder.java @@ -0,0 +1,173 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.metastore; + +import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.druid.pool.DruidDataSourceMBean; +import com.alibaba.druid.stat.JdbcDataSourceStat; +import com.alibaba.druid.stat.JdbcSqlStat; +import com.alibaba.druid.stat.JdbcSqlStatMBean; +import com.alibaba.druid.stat.JdbcStatementStat; +import com.alibaba.druid.stat.JdbcStatementStatMBean; +import io.micrometer.core.instrument.FunctionCounter; +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tags; +import io.micrometer.core.instrument.binder.BaseUnits; +import io.micrometer.core.instrument.binder.MeterBinder; +import lombok.RequiredArgsConstructor; +import org.smartdata.conf.SmartConf; + +import static org.smartdata.conf.SmartConfKeys.SMART_METRICS_DB_QUERIES_ENABLED_DEFAULT; +import static org.smartdata.conf.SmartConfKeys.SMART_METRICS_DB_QUERIES_ENABLED_KEY; + +@RequiredArgsConstructor +public class DruidPoolMetricsBinder implements MeterBinder { + private static final Tags DEFAULT_BASE_TAGS = Tags.of("component", "db"); + + private final DruidDataSource druidDataSource; + private final Tags baseTags; + private final boolean enableSqlQueryMetrics; + + public DruidPoolMetricsBinder(DruidDataSource druidDataSource, boolean enableSqlQueryMetrics) { + this(druidDataSource, DEFAULT_BASE_TAGS, enableSqlQueryMetrics); + } + + @Override + public void bindTo(MeterRegistry registry) { + FunctionCounter.builder("db.pool.druid.connections.created", + druidDataSource, DruidDataSourceMBean::getCreateCount) + .tags(baseTags) + .description("The number of created connections in the connection pool") + .register(registry); + + FunctionCounter.builder("db.pool.druid.connections.connected", + druidDataSource, DruidDataSourceMBean::getConnectCount) + .tags(baseTags) + .description("The number of opened connections in the connection pool") + .register(registry); + + FunctionCounter.builder("db.pool.druid.connections.commit.count", + druidDataSource, DruidDataSourceMBean::getCommitCount) + .tags(baseTags) + .description("The number of commits called") + .register(registry); + + FunctionCounter.builder("db.pool.druid.connections.rollback.count", + druidDataSource, DruidDataSourceMBean::getRollbackCount) + .tags(baseTags) + .description("The number of rollbacks called") + .register(registry); + + FunctionCounter.builder("db.pool.druid.connections.error.count", + druidDataSource, DruidDataSourceMBean::getErrorCount) + .tags(baseTags) + .description("The number of errors during query executions") + .register(registry); + + FunctionCounter.builder("db.pool.druid.connections.error.connect.count", + druidDataSource, DruidDataSourceMBean::getConnectErrorCount) + .tags(baseTags) + .description("The number of connection errors during query executions") + .register(registry); + + Gauge.builder("db.pool.druid.connections.active.count", + druidDataSource, DruidDataSourceMBean::getActiveCount) + .tags(baseTags) + .description("The number of active connections in the connection pool") + .register(registry); + + Gauge.builder("db.pool.druid.connections.pooling.count", + druidDataSource, DruidDataSourceMBean::getPoolingCount) + .tags(baseTags) + .description("The number of idle connections in the connection pool") + .register(registry); + + Gauge.builder("db.pool.druid.lock.wait.queue.size", + druidDataSource, DruidDataSourceMBean::getNotEmptyWaitThreadCount) + .tags(baseTags) + .description("The size of threads waiting for connection from pool") + .register(registry); + + Gauge.builder("db.pool.druid.lock.wait.time", + druidDataSource, DruidDataSourceMBean::getNotEmptyWaitMillis) + .tags(baseTags) + .description("The total time of threads waiting for connection from pool") + .baseUnit(BaseUnits.MILLISECONDS) + .register(registry); + + JdbcDataSourceStat dataSourceStat = druidDataSource.getDataSourceStat(); + + bindTo(dataSourceStat.getStatementStat(), registry); + + if (enableSqlQueryMetrics) { + dataSourceStat.getSqlStatMap() + .values() + .forEach(stat -> bindTo(stat, registry)); + } + } + + private void bindTo(JdbcSqlStat jdbcSqlStat, MeterRegistry registry) { + Tags tags = Tags.concat(baseTags, "query", jdbcSqlStat.getSql()); + + FunctionCounter.builder("db.query.execution.success.count", + jdbcSqlStat, JdbcSqlStatMBean::getExecuteSuccessCount) + .tags(tags) + .description("The number of query executions") + .register(registry); + + FunctionCounter.builder("db.query.execution.error.count", + jdbcSqlStat, JdbcSqlStatMBean::getErrorCount) + .tags(tags) + .description("The number of failing query executions") + .register(registry); + + FunctionCounter.builder("db.query.execution.running.count", + jdbcSqlStat, JdbcSqlStatMBean::getRunningCount) + .tags(tags) + .description("The number of running query executions") + .register(registry); + } + + private void bindTo(JdbcStatementStat statementStat, MeterRegistry registry) { + FunctionCounter.builder("db.pool.druid.statements.prepare.count", + statementStat, JdbcStatementStatMBean::getPrepareCallCount) + .tags(baseTags) + .description("The number of prepared statements prepare method calls") + .register(registry); + + FunctionCounter.builder("db.pool.druid.statements.execute.count", + statementStat, JdbcStatementStatMBean::getPrepareCallCount) + .tags(baseTags) + .description("The number of prepared statements executions") + .register(registry); + + Gauge.builder("db.pool.druid.statements.running.count", + statementStat, JdbcStatementStatMBean::getRunningCount) + .tags(baseTags) + .description("The number of running prepared statements") + .register(registry); + } + + public static DruidPoolMetricsBinder build(SmartConf conf, DruidDataSource druidDataSource) { + boolean enableSqlQueryMetrics = conf.getBoolean( + SMART_METRICS_DB_QUERIES_ENABLED_KEY, + SMART_METRICS_DB_QUERIES_ENABLED_DEFAULT); + return new DruidPoolMetricsBinder(druidDataSource, enableSqlQueryMetrics); + } +} diff --git a/smart-metastore/src/main/java/org/smartdata/metastore/MetaStore.java b/smart-metastore/src/main/java/org/smartdata/metastore/MetaStore.java index 0bcd23b76a4..913833a3717 100644 --- a/smart-metastore/src/main/java/org/smartdata/metastore/MetaStore.java +++ b/smart-metastore/src/main/java/org/smartdata/metastore/MetaStore.java @@ -77,8 +77,6 @@ import org.springframework.dao.EmptyResultDataAccessException; import org.springframework.transaction.PlatformTransactionManager; -import javax.sql.DataSource; - import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -202,8 +200,8 @@ public PlatformTransactionManager transactionManager() { return defaultTransactionRunner.getTransactionManager(); } - public DataSource getDataSource() { - return dbPool.getDataSource(); + public DBPool dbPool() { + return dbPool; } public Long queryForLong(String sql) throws MetaStoreException { diff --git a/smart-metastore/src/main/java/org/smartdata/metastore/utils/MetaStoreUtils.java b/smart-metastore/src/main/java/org/smartdata/metastore/utils/MetaStoreUtils.java index aa6f202a717..d57f29c9649 100644 --- a/smart-metastore/src/main/java/org/smartdata/metastore/utils/MetaStoreUtils.java +++ b/smart-metastore/src/main/java/org/smartdata/metastore/utils/MetaStoreUtils.java @@ -112,7 +112,7 @@ public static MetaStore getDBAdapter( properties = loadDefaultDruidConfig(conf, cpConfigFile); } - DruidPool druidPool = new DruidPool(properties); + DruidPool druidPool = new DruidPool(conf, properties); DBType dbType = getDbType(druidPool); PlatformTransactionManager transactionManager = diff --git a/smart-metastore/src/test/java/org/smartdata/metastore/TestDaoBase.java b/smart-metastore/src/test/java/org/smartdata/metastore/TestDaoBase.java index f0b39d24702..6599825bd63 100644 --- a/smart-metastore/src/test/java/org/smartdata/metastore/TestDaoBase.java +++ b/smart-metastore/src/test/java/org/smartdata/metastore/TestDaoBase.java @@ -26,6 +26,7 @@ import org.junit.runners.Parameterized.BeforeParam; import org.junit.runners.Parameterized.Parameter; import org.junit.runners.Parameterized.Parameters; +import org.smartdata.conf.SmartConf; import org.smartdata.metastore.dao.DaoProvider; import org.smartdata.metastore.dao.DaoProviderFactory; import org.smartdata.metastore.db.DBHandlersFactory; @@ -77,7 +78,7 @@ public static void initDao(DBType dbType, String driverClassName, String dbUrl) DBHandlersFactory dbHandlersFactory = new DBHandlersFactory(); - druidPool = new DruidPool(druidProps); + druidPool = new DruidPool(new SmartConf(), druidProps); dbSchemaManager = dbHandlersFactory .createDbManager(druidPool, new Configuration()); PlatformTransactionManager transactionManager = diff --git a/smart-metastore/src/test/java/org/smartdata/metastore/TestDruid.java b/smart-metastore/src/test/java/org/smartdata/metastore/TestDruid.java index 5696f794f33..0a0e6a75565 100644 --- a/smart-metastore/src/test/java/org/smartdata/metastore/TestDruid.java +++ b/smart-metastore/src/test/java/org/smartdata/metastore/TestDruid.java @@ -20,6 +20,7 @@ import org.apache.hadoop.conf.Configuration; import org.junit.Assert; import org.junit.Test; +import org.smartdata.conf.SmartConf; import org.smartdata.metastore.dao.DaoProvider; import org.smartdata.metastore.dao.postgres.PostgresDaoProvider; import org.smartdata.metastore.db.DBHandlersFactory; @@ -47,7 +48,7 @@ public void test() throws Exception { p.loadFromXML(in); p.setProperty(PROP_URL, "jdbc:tc:postgresql:12.19:///ssm_postgres"); p.setProperty(PROP_DRIVERCLASSNAME, ContainerDatabaseDriver.class.getName()); - DruidPool druidPool = new DruidPool(p); + DruidPool druidPool = new DruidPool(new SmartConf(), p); DBHandlersFactory dbHandlersFactory = new DBHandlersFactory(); DbSchemaManager dbSchemaManager = dbHandlersFactory .createDbManager(druidPool, new Configuration()); diff --git a/smart-metrics/README.md b/smart-metrics/README.md new file mode 100644 index 00000000000..47ca6dc63cb --- /dev/null +++ b/smart-metrics/README.md @@ -0,0 +1,69 @@ +# Supported metrics + +| Name | Description | +|-----------------------------------------------------|---------------------------------------------------------------------------------------------------------| +| access_events_count_total | Access events count | +| application_ready_time_seconds | Time taken for the application to be ready to service requests | +| application_started_time_seconds | Time taken to start the application | +| db_pool_druid_connections_active_count | The number of active connections in the connection pool | +| db_pool_druid_connections_commit_count_total | The number of commits called | +| db_pool_druid_connections_connected_total | The number of opened connections in the connection pool | +| db_pool_druid_connections_created_total | The number of created connections in the connection pool | +| db_pool_druid_connections_error_connect_count_total | The number of connection errors during query executions | +| db_pool_druid_connections_error_count_total | The number of errors during query executions | +| db_pool_druid_connections_pooling_count | The number of idle connections in the connection pool | +| db_pool_druid_connections_rollback_count_total | The number of rollbacks called | +| db_pool_druid_lock_wait_queue_size | The size of threads waiting for connection from pool | +| db_pool_druid_lock_wait_time_ms | The total time of threads waiting for connection from pool | +| db_pool_druid_statements_execute_count_total | The number of prepared statements executions | +| db_pool_druid_statements_prepare_count_total | The number of prepared statements prepare method calls | +| db_pool_druid_statements_running_count | The number of running prepared statements | +| executor_active_threads | The approximate number of threads that are actively executing tasks | +| executor_completed_tasks_total | The approximate total number of tasks that have completed execution | +| executor_idle_seconds | Executor idle time | +| executor_idle_seconds_max | Maximum idle time for executor | +| executor_pool_core_threads | The core number of threads for the pool | +| executor_pool_max_threads | The maximum allowed number of threads in the pool | +| executor_pool_size_threads | The current number of threads in the pool | +| executor_queue_remaining_tasks | The number of additional elements that this queue can ideally accept without blocking | +| executor_queued_tasks | The approximate number of tasks that are queued for execution | +| executor_scheduled_once_total | The total number of tasks scheduled once | +| executor_scheduled_repetitively_total | The total number of tasks scheduled repetitively | +| executor_seconds | Total time spent by executor | +| executor_seconds_max | Maximum time spent by executor | +| http_server_requests_seconds | Duration of HTTP server request handling | +| http_server_requests_seconds_max | Maximum duration of HTTP server request handling | +| jvm_buffer_count_buffers | An estimate of the number of buffers in the pool | +| jvm_buffer_memory_used_bytes | An estimate of the memory that the Java virtual machine is using for this buffer pool | +| jvm_buffer_total_capacity_bytes | An estimate of the total capacity of the buffers in this pool | +| jvm_classes_loaded_classes | The number of classes that are currently loaded in the Java virtual machine | +| jvm_classes_unloaded_classes_total | The total number of classes unloaded since the Java virtual machine has started execution | +| jvm_gc_live_data_size_bytes | Size of long-lived heap memory pool after reclamation | +| jvm_gc_max_data_size_bytes | Max size of long-lived heap memory pool | +| jvm_gc_memory_allocated_bytes_total | Incremented for an increase in the size of the (young) heap memory pool after one GC to before the next | +| jvm_gc_memory_promoted_bytes_total | Count of positive increases in the size of the old generation memory pool before GC to after GC | +| jvm_gc_overhead_percent | Percent of CPU time used by GC activities over the last lookback period or since monitoring began | +| jvm_gc_pause_seconds | Time spent in GC pause | +| jvm_gc_pause_seconds_max | Maximum time spent in GC pause | +| jvm_memory_committed_bytes | The amount of memory in bytes that is committed for the Java virtual machine to use | +| jvm_memory_max_bytes | The maximum amount of memory in bytes that can be used for memory management | +| jvm_memory_usage_after_gc_percent | The percentage of long-lived heap pool used after the last GC event, in the range [0..1] | +| jvm_memory_used_bytes | The amount of used memory | +| jvm_threads_daemon_threads | The current number of live daemon threads | +| jvm_threads_live_threads | The current number of live threads including both daemon and non-daemon threads | +| jvm_threads_peak_threads | The peak live thread count since the Java virtual machine started or peak was reset | +| jvm_threads_states_threads | The current number of threads in each state | +| process_cpu_usage | The recent CPU usage for the Java Virtual Machine process | +| process_files_max_files | The maximum file descriptor count | +| process_files_open_files | The open file descriptor count | +| process_start_time_seconds | Start time of the process since Unix epoch | +| process_uptime_seconds | The uptime of the Java virtual machine | +| system_cpu_count | The number of processors available to the Java virtual machine | +| system_cpu_usage | The recent CPU usage of the system the application is running in | +| system_load_average_1m | The sum of runnable entities queued and running on available processors averaged over a period of time | +| tomcat_sessions_active_current_sessions | The number of current active sessions | +| tomcat_sessions_active_max_sessions | The maximum number of active sessions | +| tomcat_sessions_alive_max_seconds | The maximum number of seconds a session was alive | +| tomcat_sessions_created_sessions_total | The total number of sessions created | +| tomcat_sessions_expired_sessions_total | The total number of expired sessions | +| tomcat_sessions_rejected_sessions_total | The total number of rejected sessions | diff --git a/smart-metrics/pom.xml b/smart-metrics/pom.xml index 7aec1c9c9f7..60c6bb2f2f9 100644 --- a/smart-metrics/pom.xml +++ b/smart-metrics/pom.xml @@ -35,34 +35,27 @@ - org.apache.hadoop - hadoop-common - - - org.eclipse.jetty - jetty-server - - - org.eclipse.jetty - jetty-servlet - - - org.eclipse.jetty - jetty-webapp - - - log4j - log4j - - - ch.qos.reload4j - reload4j - - - org.slf4j - slf4j-reload4j - - + org.smartdata + smart-common + 2.0.0-SNAPSHOT + + + org.projectlombok + lombok + provided + + + io.micrometer + micrometer-registry-prometheus + + + io.micrometer + micrometer-registry-jmx + + + io.prometheus + prometheus-metrics-exporter-httpserver + 1.3.1 diff --git a/smart-metrics/src/main/java/org/smartdata/metrics/CompositeMetricsFactory.java b/smart-metrics/src/main/java/org/smartdata/metrics/CompositeMetricsFactory.java new file mode 100644 index 00000000000..41a2d4f4adf --- /dev/null +++ b/smart-metrics/src/main/java/org/smartdata/metrics/CompositeMetricsFactory.java @@ -0,0 +1,162 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.metrics; + +import io.micrometer.core.instrument.Clock; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tags; +import io.micrometer.core.instrument.binder.MeterBinder; +import io.micrometer.core.instrument.binder.jvm.ClassLoaderMetrics; +import io.micrometer.core.instrument.binder.jvm.ExecutorServiceMetrics; +import io.micrometer.core.instrument.binder.jvm.JvmGcMetrics; +import io.micrometer.core.instrument.binder.jvm.JvmHeapPressureMetrics; +import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics; +import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics; +import io.micrometer.core.instrument.binder.system.FileDescriptorMetrics; +import io.micrometer.core.instrument.binder.system.ProcessorMetrics; +import io.micrometer.core.instrument.binder.system.UptimeMetrics; +import io.micrometer.core.instrument.composite.CompositeMeterRegistry; +import io.micrometer.jmx.JmxConfig; +import io.micrometer.jmx.JmxMeterRegistry; +import io.micrometer.prometheus.PrometheusConfig; +import io.micrometer.prometheus.PrometheusMeterRegistry; +import lombok.extern.slf4j.Slf4j; +import org.smartdata.conf.SmartConf; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.ScheduledExecutorService; +import java.util.stream.Collectors; + +import static org.smartdata.conf.SmartConfKeys.SMART_METRICS_JMX_DOMAIN_DEFAULT; +import static org.smartdata.conf.SmartConfKeys.SMART_METRICS_JMX_DOMAIN_KEY; +import static org.smartdata.conf.SmartConfKeys.SMART_METRICS_JMX_ENABLED_DEFAULT; +import static org.smartdata.conf.SmartConfKeys.SMART_METRICS_JMX_ENABLED_KEY; +import static org.smartdata.conf.SmartConfKeys.SMART_METRICS_PROMETHEUS_ENABLED_DEFAULT; +import static org.smartdata.conf.SmartConfKeys.SMART_METRICS_PROMETHEUS_ENABLED_KEY; + +@Slf4j +public class CompositeMetricsFactory implements MetricsFactory { + private final CompositeMeterRegistry compositeMeterRegistry; + + private final Iterable resources; + private final Tags baseTags; + + public CompositeMetricsFactory(Iterable registries, Tags baseTags) { + this( + new CompositeMeterRegistry(Clock.SYSTEM, registries), + Arrays.asList( + new ProcessorMetrics(), + new UptimeMetrics(), + new JvmGcMetrics(), + new JvmMemoryMetrics(), + new FileDescriptorMetrics(), + new ClassLoaderMetrics(), + new JvmHeapPressureMetrics(), + new JvmThreadMetrics()), + baseTags); + } + + public CompositeMetricsFactory(CompositeMeterRegistry compositeRegistry, + Collection defaultMetrics, + Tags baseTags) { + this.compositeMeterRegistry = compositeRegistry; + this.baseTags = baseTags; + this.resources = defaultMetrics.stream() + .filter(metric -> metric instanceof AutoCloseable) + .map(AutoCloseable.class::cast) + .collect(Collectors.toList()); + defaultMetrics + .forEach(metric -> metric.bindTo(compositeMeterRegistry)); + } + + @Override + public CompositeMeterRegistry getMeterRegistry() { + return compositeMeterRegistry; + } + + @Override + public Counter counter(String name, String... tags) { + return compositeMeterRegistry.counter(name, Tags.concat(baseTags, tags)); + } + + @Override + public ScheduledExecutorService wrap(ScheduledExecutorService executorService, String name, String... tags) { + return ExecutorServiceMetrics.monitor(compositeMeterRegistry, + executorService, name, Tags.concat(baseTags, tags)); + } + + @Override + public void close() throws Exception { + for (AutoCloseable closeable : resources) { + try { + closeable.close(); + } catch (Exception e) { + log.error("Error closing resource", e); + } + } + } + + public static MetricsFactory from(SmartConf conf, Tags baseTags) { + List registries = new ArrayList<>(); + + boolean prometheusEnabled = conf.getBoolean( + SMART_METRICS_PROMETHEUS_ENABLED_KEY, SMART_METRICS_PROMETHEUS_ENABLED_DEFAULT); + if (prometheusEnabled) { + registries.add(new PrometheusMeterRegistry(PrometheusConfig.DEFAULT)); + } + + boolean jmxEnabled = conf.getBoolean( + SMART_METRICS_JMX_ENABLED_KEY, SMART_METRICS_JMX_ENABLED_DEFAULT); + if (jmxEnabled) { + registries.add(new JmxMeterRegistry(new SsmJmxConfig(conf), Clock.SYSTEM)); + } + + if (registries.isEmpty()) { + log.warn("No metrics exporters are provided, although metrics collection is enabled. " + + "Switching to no-op implementation"); + return MetricsFactory.noOp(); + } + + return new CompositeMetricsFactory(registries, baseTags); + } + + private static class SsmJmxConfig implements JmxConfig { + + private final String domain; + + private SsmJmxConfig(SmartConf conf) { + this.domain = conf.get( + SMART_METRICS_JMX_DOMAIN_KEY, + SMART_METRICS_JMX_DOMAIN_DEFAULT); + } + + public String domain() { + return domain; + } + + @Override + public String get(String key) { + return null; + } + } + +} diff --git a/smart-metrics/src/main/java/org/smartdata/metrics/MetricsFactory.java b/smart-metrics/src/main/java/org/smartdata/metrics/MetricsFactory.java new file mode 100644 index 00000000000..61de6e3a418 --- /dev/null +++ b/smart-metrics/src/main/java/org/smartdata/metrics/MetricsFactory.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.metrics; + +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.Tags; +import io.micrometer.core.instrument.composite.CompositeMeterRegistry; +import org.smartdata.conf.SmartConf; + +import java.util.concurrent.ScheduledExecutorService; + +import static org.smartdata.conf.SmartConfKeys.SMART_METRICS_ENABLED_DEFAULT; +import static org.smartdata.conf.SmartConfKeys.SMART_METRICS_ENABLED_KEY; + +public interface MetricsFactory extends AutoCloseable { + + CompositeMeterRegistry getMeterRegistry(); + + Counter counter(String name, String... tags); + + ScheduledExecutorService wrap(ScheduledExecutorService executorService, String name, String... tags); + + static MetricsFactory noOp() { + return new NoOpMetricsFactory(); + } + + static MetricsFactory from(SmartConf conf, Tags baseTags) { + return conf.getBoolean(SMART_METRICS_ENABLED_KEY, SMART_METRICS_ENABLED_DEFAULT) + ? CompositeMetricsFactory.from(conf, baseTags) + : MetricsFactory.noOp(); + } +} diff --git a/smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/user/SsmLdapUserSearch.java b/smart-metrics/src/main/java/org/smartdata/metrics/MetricsFactoryHolder.java similarity index 72% rename from smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/user/SsmLdapUserSearch.java rename to smart-metrics/src/main/java/org/smartdata/metrics/MetricsFactoryHolder.java index 52f47700ea5..45bd1f638f5 100644 --- a/smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/user/SsmLdapUserSearch.java +++ b/smart-metrics/src/main/java/org/smartdata/metrics/MetricsFactoryHolder.java @@ -15,12 +15,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.config.ldap.search.user; +package org.smartdata.metrics; -import org.smartdata.server.config.ldap.search.query.LdapExpressionTemplate; -import org.springframework.security.ldap.search.LdapUserSearch; +public class MetricsFactoryHolder { + private static volatile MetricsFactory metricsFactory; -public interface SsmLdapUserSearch extends LdapUserSearch { + public static void set(MetricsFactory metricsFactory) { + MetricsFactoryHolder.metricsFactory = metricsFactory; + } - LdapExpressionTemplate getSearchTemplate(); + public static MetricsFactory get() { + return metricsFactory; + } } diff --git a/smart-metrics/src/main/java/org/smartdata/metrics/NoOpMetricsFactory.java b/smart-metrics/src/main/java/org/smartdata/metrics/NoOpMetricsFactory.java new file mode 100644 index 00000000000..2ce34f2b27c --- /dev/null +++ b/smart-metrics/src/main/java/org/smartdata/metrics/NoOpMetricsFactory.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.metrics; + +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.Meter; +import io.micrometer.core.instrument.Tags; +import io.micrometer.core.instrument.composite.CompositeMeterRegistry; +import io.micrometer.core.instrument.noop.NoopCounter; +import lombok.extern.slf4j.Slf4j; + +import java.util.concurrent.ScheduledExecutorService; + + +@Slf4j +public class NoOpMetricsFactory implements MetricsFactory { + + private final Counter counter; + + public NoOpMetricsFactory() { + this.counter = new NoopCounter(id("no-op-counter")); + } + + @Override + public CompositeMeterRegistry getMeterRegistry() { + return new CompositeMeterRegistry(); + } + + @Override + public Counter counter(String name, String... tags) { + return counter; + } + + @Override + public ScheduledExecutorService wrap(ScheduledExecutorService executorService, String name, String... tags) { + return executorService; + } + + @Override + public void close() throws Exception { + + } + + private Meter.Id id(String name) { + return new Meter.Id(name, Tags.empty(), null, null, Meter.Type.OTHER); + } +} diff --git a/smart-server/pom.xml b/smart-server/pom.xml index 90380ba0fe2..564dae53d7e 100644 --- a/smart-server/pom.xml +++ b/smart-server/pom.xml @@ -47,12 +47,7 @@ org.smartdata - smart-metrics - 2.0.0-SNAPSHOT - - - org.smartdata - smart-web-server + smart-master-web-server 2.0.0-SNAPSHOT diff --git a/smart-server/src/main/java/org/smartdata/server/SmartServer.java b/smart-server/src/main/java/org/smartdata/server/SmartServer.java index 66243300fad..c8f838422b2 100644 --- a/smart-server/src/main/java/org/smartdata/server/SmartServer.java +++ b/smart-server/src/main/java/org/smartdata/server/SmartServer.java @@ -17,6 +17,7 @@ */ package org.smartdata.server; +import io.micrometer.core.instrument.Tags; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.Option; @@ -30,7 +31,9 @@ import org.smartdata.conf.SmartConf; import org.smartdata.conf.SmartConfKeys; import org.smartdata.hdfs.HadoopUtil; +import org.smartdata.http.SmartHttpServer; import org.smartdata.metastore.MetaStore; +import org.smartdata.metrics.MetricsFactory; import org.smartdata.server.cluster.ClusterNodesManager; import org.smartdata.server.engine.CmdletManager; import org.smartdata.server.engine.RuleManager; @@ -54,13 +57,14 @@ */ public class SmartServer { public static final Logger LOG = LoggerFactory.getLogger(SmartServer.class); + public static final Tags SMART_SERVER_BASE_TAGS = Tags.of("service", "smart-server"); private final SmartConf conf; private SmartEngine engine; private ServerContext context; private volatile boolean enabled; private SmartRpcServer rpcServer; - private SmartRestServer restServer; + private SmartHttpServer restServer; static { SLF4JBridgeHandler.removeHandlersForRootLogger(); @@ -76,10 +80,15 @@ public void initWith(MetaStore metaStore) throws Exception { LOG.info("Start Init Smart Server"); HadoopUtil.setSmartConfByHadoop(conf); - context = new ServerContext(conf, metaStore); + + MetricsFactory metricsFactory = MetricsFactory.from(conf, SMART_SERVER_BASE_TAGS); + metaStore.dbPool().bindMetrics(metricsFactory); + + context = new ServerContext(conf, metaStore, metricsFactory); engine = new SmartEngine(context); + // TODO add RPC metrics rpcServer = new SmartRpcServer(this, conf); - restServer = new SmartRestServer(conf, engine); + restServer = new SmartMasterRestServer(conf, engine); LOG.info("Finish Init Smart Server"); } @@ -122,7 +131,7 @@ private static StartupOption processArgs(String[] args, SmartConf conf) throws E } else if (StartupOption.REGULAR.getName().equalsIgnoreCase(arg)) { startOpt = StartupOption.REGULAR; } else if (arg.equals("-h") || arg.equals("-help")) { - if (parseHelpArgument(new String[] {arg}, USAGE, System.out)) { + if (parseHelpArgument(new String[]{arg}, USAGE, System.out)) { return null; } } else { diff --git a/smart-server/src/main/resources/hadoop-metrics2-ssm.properties b/smart-server/src/main/resources/hadoop-metrics2-ssm.properties new file mode 100644 index 00000000000..9c6de578cf1 --- /dev/null +++ b/smart-server/src/main/resources/hadoop-metrics2-ssm.properties @@ -0,0 +1,5 @@ +# syntax: [prefix].[source|sink].[instance].[options] +# See javadoc of package-info.java for org.apache.hadoop.metrics2 for details + +*.source.filter.class=org.apache.hadoop.metrics2.filter.GlobFilter +ssm.sink.rpcServerMetricsExporter.record.filter.include=rpc* \ No newline at end of file diff --git a/smart-server/src/test/java/org/smartdata/server/engine/TestCmdletManager.java b/smart-server/src/test/java/org/smartdata/server/engine/TestCmdletManager.java index 946cfb2d254..068c4054442 100644 --- a/smart-server/src/test/java/org/smartdata/server/engine/TestCmdletManager.java +++ b/smart-server/src/test/java/org/smartdata/server/engine/TestCmdletManager.java @@ -294,7 +294,7 @@ private void safeSubmit(String cmdlet) { } private void flushToDB(MetaStore metaStore, - List actionInfos, CmdletInfo cmdletInfo) throws Exception { + List actionInfos, CmdletInfo cmdletInfo) throws Exception { for (ActionInfo actionInfo : actionInfos) { cmdletInfo.addAction(actionInfo.getActionId()); } diff --git a/smart-web-server/pom.xml b/smart-web-server/pom.xml index a8f6ea30a73..145f233a48e 100644 --- a/smart-web-server/pom.xml +++ b/smart-web-server/pom.xml @@ -29,13 +29,19 @@ smart-web-server + pom + + smart-web-server-base + smart-master-web-server + smart-agent-web-server + ssm-api.yaml ${project.basedir}/src/main/resources/static ${project.basedir} org.smartdata.server.generated - ../smart-frontend/build/ + ../../smart-frontend/build/ @@ -57,6 +63,12 @@ 2.0.0-SNAPSHOT + + org.smartdata + smart-metrics + 2.0.0-SNAPSHOT + + org.springframework.boot spring-boot-starter-web @@ -189,132 +201,8 @@ - - org.apache.maven.plugins - maven-compiler-plugin - - - - org.mapstruct - mapstruct-processor - ${mapstruct.version} - - - org.projectlombok - lombok - ${lombok.version} - - - - - - maven-resources-plugin - ${maven-resources-plugin.version} - - - copy-resources - compile - - copy-resources - - - ${skip-build-web-ui} - ${project.build.directory}/classes/static - - - ${ui.assets.directory} - false - - **/*.woff2 - **/*.ttf - - - - ${ui.assets.directory} - true - - **/*.woff2 - **/*.ttf - - - - - - - - - - web-server-codegen - - - - org.openapitools - openapi-generator-maven-plugin - ${openapi.generator.version} - - - generate-merged-api-spec - - generate - - - ${project.basedir}/src/main/resources/api/ssm-api.yaml - openapi-yaml - - ${merged.api.spec.name} - - ${merged.api.spec.directory} - ${merged.api.spec.name} - - - - generate-ssm-web-server - - generate - - - ${merged.api.spec.directory}/${merged.api.spec.name} - spring - - false - false - true - true - true - true - ${openapi.codegen.package.base}.config - - ${openapi.codegen.output.dir} - ${openapi.codegen.package.base}.model - ${openapi.codegen.package.base}.api - Dto - false - false - true - true - - - - - - - - - - - java11 - - 7.3.0 - - - \ No newline at end of file diff --git a/smart-web-server/smart-agent-web-server/pom.xml b/smart-web-server/smart-agent-web-server/pom.xml new file mode 100644 index 00000000000..ad7c028104e --- /dev/null +++ b/smart-web-server/smart-agent-web-server/pom.xml @@ -0,0 +1,40 @@ + + + + 4.0.0 + + org.smartdata + smart-web-server + 2.0.0-SNAPSHOT + + + smart-agent-web-server + + + + org.smartdata + smart-web-server-base + 2.0.0-SNAPSHOT + + + \ No newline at end of file diff --git a/smart-web-server/smart-agent-web-server/src/main/java/org/smartdata/agent/http/SmartAgentHttpServer.java b/smart-web-server/smart-agent-web-server/src/main/java/org/smartdata/agent/http/SmartAgentHttpServer.java new file mode 100644 index 00000000000..d3500e832c2 --- /dev/null +++ b/smart-web-server/smart-agent-web-server/src/main/java/org/smartdata/agent/http/SmartAgentHttpServer.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.agent.http; + +import lombok.RequiredArgsConstructor; +import org.smartdata.conf.SmartConf; +import org.smartdata.http.SmartHttpServer; +import org.smartdata.metrics.MetricsFactory; +import org.smartdata.security.SmartPrincipalManager; +import org.springframework.beans.factory.config.ConfigurableListableBeanFactory; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.context.ApplicationContextInitializer; +import org.springframework.context.ConfigurableApplicationContext; + +import static org.smartdata.conf.SmartConfKeys.SMART_AGENT_HTTP_SERVER_PORT_DEFAULT; +import static org.smartdata.conf.SmartConfKeys.SMART_AGENT_HTTP_SERVER_PORT_KEY; + +public class SmartAgentHttpServer extends SmartHttpServer { + public static final String AGENT_PROFILE = "agent"; + + public SmartAgentHttpServer(SmartConf ssmConfig, MetricsFactory metricsFactory) { + super(ssmConfig, new ContextInitializer(metricsFactory), getServerPort(ssmConfig)); + } + + @RequiredArgsConstructor + private static class ContextInitializer implements + ApplicationContextInitializer { + + private final MetricsFactory metricsFactory; + + @Override + public void initialize(ConfigurableApplicationContext applicationContext) { + ConfigurableListableBeanFactory beanFactory = applicationContext.getBeanFactory(); + beanFactory.registerSingleton("metricsFactory", metricsFactory); + beanFactory.registerSingleton("smartPrincipalManager", + SmartPrincipalManager.noOp()); + } + } + + @Override + protected SpringApplicationBuilder customizeSpringApplication(SpringApplicationBuilder builder) { + return builder.profiles(AGENT_PROFILE); + } + + private static int getServerPort(SmartConf conf) { + return conf.getInt( + SMART_AGENT_HTTP_SERVER_PORT_KEY, + SMART_AGENT_HTTP_SERVER_PORT_DEFAULT); + } +} diff --git a/smart-web-server/smart-agent-web-server/src/main/resources/application-agent.yaml b/smart-web-server/smart-agent-web-server/src/main/resources/application-agent.yaml new file mode 100644 index 00000000000..66d923af51b --- /dev/null +++ b/smart-web-server/smart-agent-web-server/src/main/resources/application-agent.yaml @@ -0,0 +1,5 @@ +management: + endpoints: + web: + exposure: + include: health,prometheus diff --git a/smart-web-server/.openapi-generator-ignore b/smart-web-server/smart-master-web-server/.openapi-generator-ignore similarity index 100% rename from smart-web-server/.openapi-generator-ignore rename to smart-web-server/smart-master-web-server/.openapi-generator-ignore diff --git a/smart-web-server/smart-master-web-server/pom.xml b/smart-web-server/smart-master-web-server/pom.xml new file mode 100644 index 00000000000..865d0e59642 --- /dev/null +++ b/smart-web-server/smart-master-web-server/pom.xml @@ -0,0 +1,171 @@ + + + + 4.0.0 + + org.smartdata + smart-web-server + 2.0.0-SNAPSHOT + + + smart-master-web-server + + + + org.smartdata + smart-web-server-base + 2.0.0-SNAPSHOT + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + + org.mapstruct + mapstruct-processor + ${mapstruct.version} + + + org.projectlombok + lombok + ${lombok.version} + + + + + + maven-resources-plugin + ${maven-resources-plugin.version} + + + copy-resources + compile + + copy-resources + + + ${skip-build-web-ui} + ${project.build.directory}/classes/static + + + ${ui.assets.directory} + false + + **/*.woff2 + **/*.ttf + + + + ${ui.assets.directory} + true + + **/*.woff2 + **/*.ttf + + + + + + + + + + + + + web-server-codegen + + + + org.openapitools + openapi-generator-maven-plugin + ${openapi.generator.version} + + + generate-merged-api-spec + + generate + + + ${project.basedir}/src/main/resources/api/ssm-api.yaml + openapi-yaml + + ${merged.api.spec.name} + + ${merged.api.spec.directory} + ${merged.api.spec.name} + + + + generate-ssm-web-server + + generate + + + ${merged.api.spec.directory}/${merged.api.spec.name} + spring + + false + false + true + true + true + true + ${openapi.codegen.package.base}.config + + ${openapi.codegen.output.dir} + ${openapi.codegen.package.base}.model + ${openapi.codegen.package.base}.api + Dto + false + false + true + true + + + + + + + + + + + java11 + + 7.3.0 + + + + + \ No newline at end of file diff --git a/smart-web-server/src/main/java/org/smartdata/server/config/SsmContextInitializer.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/SmartMasterRestServer.java similarity index 64% rename from smart-web-server/src/main/java/org/smartdata/server/config/SsmContextInitializer.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/SmartMasterRestServer.java index 6f0fd80fae1..21f7624276d 100644 --- a/smart-web-server/src/main/java/org/smartdata/server/config/SsmContextInitializer.java +++ b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/SmartMasterRestServer.java @@ -15,32 +15,44 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.config; +package org.smartdata.server; import lombok.RequiredArgsConstructor; import org.smartdata.conf.SmartConf; -import org.smartdata.server.SmartEngine; +import org.smartdata.http.SmartHttpServer; import org.springframework.beans.factory.config.ConfigurableListableBeanFactory; +import org.springframework.boot.builder.SpringApplicationBuilder; import org.springframework.context.ApplicationContextInitializer; import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.ComponentScan; -@RequiredArgsConstructor -public class SsmContextInitializer implements - ApplicationContextInitializer { +import static org.smartdata.conf.SmartConfKeys.SMART_REST_SERVER_PORT_KEY; +import static org.smartdata.conf.SmartConfKeys.SMART_REST_SERVER_PORT_KEY_DEFAULT; - private final SmartEngine smartEngine; +public class SmartMasterRestServer extends SmartHttpServer { + public static final String SSM_MASTER_PROFILE = "master"; + + public SmartMasterRestServer(SmartConf ssmConfig, SmartEngine smartEngine) { + super(ssmConfig, new ContextInitializer(smartEngine), + getServerPort(ssmConfig), EnableComponentScan.class); + } - private final SmartConf conf; + @RequiredArgsConstructor + private static class ContextInitializer implements + ApplicationContextInitializer { + + private final SmartEngine smartEngine; @Override public void initialize(ConfigurableApplicationContext applicationContext) { ConfigurableListableBeanFactory beanFactory = applicationContext.getBeanFactory(); - beanFactory.registerSingleton("smartConfig", conf); beanFactory.registerSingleton("smartEngine", smartEngine); beanFactory.registerSingleton("statesManager", smartEngine.getStatesManager()); beanFactory.registerSingleton("cmdletManager", smartEngine.getCmdletManager()); beanFactory.registerSingleton("ruleManager", smartEngine.getRuleManager()); beanFactory.registerSingleton("auditService", smartEngine.getAuditService()); + beanFactory.registerSingleton("metricsFactory", + smartEngine.getServerContext().getMetricsFactory()); beanFactory.registerSingleton("clusterNodesManager", smartEngine.getClusterNodesManager()); beanFactory.registerSingleton( @@ -54,4 +66,18 @@ public void initialize(ConfigurableApplicationContext applicationContext) { beanFactory.registerSingleton("dbFileAccessManager", smartEngine.getStatesManager().getFileAccessManager()); } + } + + @Override + protected SpringApplicationBuilder customizeSpringApplication(SpringApplicationBuilder builder) { + return builder.profiles(SSM_MASTER_PROFILE); + } + + private static int getServerPort(SmartConf conf) { + return conf.getInt(SMART_REST_SERVER_PORT_KEY, SMART_REST_SERVER_PORT_KEY_DEFAULT); + } + + @ComponentScan(basePackages = "org.smartdata.server") + public static class EnableComponentScan { + } } diff --git a/smart-web-server/src/main/java/org/smartdata/server/controller/ActionsControllerDelegate.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/controller/ActionsControllerDelegate.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/controller/ActionsControllerDelegate.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/controller/ActionsControllerDelegate.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/controller/AuditControllerDelegate.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/controller/AuditControllerDelegate.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/controller/AuditControllerDelegate.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/controller/AuditControllerDelegate.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/controller/CachedFilesControllerDelegate.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/controller/CachedFilesControllerDelegate.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/controller/CachedFilesControllerDelegate.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/controller/CachedFilesControllerDelegate.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/controller/ClusterControllerDelegate.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/controller/ClusterControllerDelegate.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/controller/ClusterControllerDelegate.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/controller/ClusterControllerDelegate.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/controller/CmdletControllerDelegate.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/controller/CmdletControllerDelegate.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/controller/CmdletControllerDelegate.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/controller/CmdletControllerDelegate.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/controller/FilesControllerDelegate.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/controller/FilesControllerDelegate.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/controller/FilesControllerDelegate.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/controller/FilesControllerDelegate.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/controller/RulesControllerDelegate.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/controller/RulesControllerDelegate.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/controller/RulesControllerDelegate.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/controller/RulesControllerDelegate.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/controller/SystemControllerDelegate.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/controller/SystemControllerDelegate.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/controller/SystemControllerDelegate.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/controller/SystemControllerDelegate.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/controller/WebUiController.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/controller/WebUiController.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/controller/WebUiController.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/controller/WebUiController.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/api/ActionsApi.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/ActionsApi.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/api/ActionsApi.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/ActionsApi.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/api/ActionsApiController.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/ActionsApiController.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/api/ActionsApiController.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/ActionsApiController.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/api/ActionsApiDelegate.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/ActionsApiDelegate.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/api/ActionsApiDelegate.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/ActionsApiDelegate.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/api/AuditApi.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/AuditApi.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/api/AuditApi.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/AuditApi.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/api/AuditApiController.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/AuditApiController.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/api/AuditApiController.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/AuditApiController.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/api/AuditApiDelegate.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/AuditApiDelegate.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/api/AuditApiDelegate.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/AuditApiDelegate.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/api/ClusterApi.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/ClusterApi.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/api/ClusterApi.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/ClusterApi.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/api/ClusterApiController.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/ClusterApiController.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/api/ClusterApiController.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/ClusterApiController.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/api/ClusterApiDelegate.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/ClusterApiDelegate.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/api/ClusterApiDelegate.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/ClusterApiDelegate.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/api/CmdletsApi.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/CmdletsApi.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/api/CmdletsApi.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/CmdletsApi.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/api/CmdletsApiController.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/CmdletsApiController.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/api/CmdletsApiController.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/CmdletsApiController.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/api/CmdletsApiDelegate.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/CmdletsApiDelegate.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/api/CmdletsApiDelegate.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/CmdletsApiDelegate.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/api/FilesApi.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/FilesApi.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/api/FilesApi.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/FilesApi.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/api/FilesApiController.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/FilesApiController.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/api/FilesApiController.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/FilesApiController.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/api/FilesApiDelegate.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/FilesApiDelegate.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/api/FilesApiDelegate.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/FilesApiDelegate.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/api/RulesApi.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/RulesApi.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/api/RulesApi.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/RulesApi.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/api/RulesApiController.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/RulesApiController.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/api/RulesApiController.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/RulesApiController.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/api/RulesApiDelegate.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/RulesApiDelegate.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/api/RulesApiDelegate.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/RulesApiDelegate.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/api/SystemApi.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/SystemApi.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/api/SystemApi.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/SystemApi.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/api/SystemApiController.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/SystemApiController.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/api/SystemApiController.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/SystemApiController.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/api/SystemApiDelegate.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/SystemApiDelegate.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/api/SystemApiDelegate.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/api/SystemApiDelegate.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/config/EnumConverterConfiguration.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/config/EnumConverterConfiguration.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/config/EnumConverterConfiguration.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/config/EnumConverterConfiguration.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/ActionDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/ActionDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/ActionDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/ActionDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/ActionInfoDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/ActionInfoDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/ActionInfoDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/ActionInfoDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/ActionSortDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/ActionSortDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/ActionSortDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/ActionSortDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/ActionSourceDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/ActionSourceDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/ActionSourceDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/ActionSourceDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/ActionStateDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/ActionStateDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/ActionStateDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/ActionStateDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/ActionsDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/ActionsDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/ActionsDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/ActionsDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/AuditEventDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/AuditEventDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/AuditEventDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/AuditEventDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/AuditEventResultDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/AuditEventResultDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/AuditEventResultDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/AuditEventResultDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/AuditEventsDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/AuditEventsDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/AuditEventsDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/AuditEventsDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/AuditObjectTypeDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/AuditObjectTypeDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/AuditObjectTypeDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/AuditObjectTypeDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/AuditOperationDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/AuditOperationDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/AuditOperationDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/AuditOperationDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/AuditSortDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/AuditSortDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/AuditSortDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/AuditSortDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/CachedFileInfoDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/CachedFileInfoDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/CachedFileInfoDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/CachedFileInfoDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/CachedFileSortDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/CachedFileSortDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/CachedFileSortDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/CachedFileSortDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/CachedFilesDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/CachedFilesDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/CachedFilesDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/CachedFilesDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/CachedTimeIntervalDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/CachedTimeIntervalDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/CachedTimeIntervalDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/CachedTimeIntervalDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/ClusterNodeDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/ClusterNodeDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/ClusterNodeDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/ClusterNodeDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/ClusterNodesDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/ClusterNodesDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/ClusterNodesDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/ClusterNodesDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/ClusterSortDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/ClusterSortDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/ClusterSortDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/ClusterSortDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/CmdletDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/CmdletDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/CmdletDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/CmdletDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/CmdletSortDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/CmdletSortDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/CmdletSortDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/CmdletSortDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/CmdletStateDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/CmdletStateDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/CmdletStateDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/CmdletStateDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/CmdletsDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/CmdletsDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/CmdletsDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/CmdletsDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/CompletionTimeIntervalDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/CompletionTimeIntervalDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/CompletionTimeIntervalDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/CompletionTimeIntervalDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/ErrorResponseDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/ErrorResponseDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/ErrorResponseDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/ErrorResponseDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/EventTimeIntervalDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/EventTimeIntervalDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/EventTimeIntervalDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/EventTimeIntervalDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/ExecutorTypeDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/ExecutorTypeDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/ExecutorTypeDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/ExecutorTypeDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/FileAccessCountsDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/FileAccessCountsDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/FileAccessCountsDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/FileAccessCountsDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/FileAccessInfoDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/FileAccessInfoDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/FileAccessInfoDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/FileAccessInfoDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/HotFileSortDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/HotFileSortDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/HotFileSortDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/HotFileSortDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/LastAccessedTimeIntervalDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/LastAccessedTimeIntervalDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/LastAccessedTimeIntervalDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/LastAccessedTimeIntervalDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/LastActivationTimeIntervalDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/LastActivationTimeIntervalDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/LastActivationTimeIntervalDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/LastActivationTimeIntervalDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/PageRequestDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/PageRequestDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/PageRequestDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/PageRequestDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/RegistrationTimeIntervalDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/RegistrationTimeIntervalDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/RegistrationTimeIntervalDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/RegistrationTimeIntervalDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/RuleDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/RuleDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/RuleDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/RuleDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/RuleSortDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/RuleSortDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/RuleSortDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/RuleSortDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/RuleStateDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/RuleStateDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/RuleStateDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/RuleStateDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/RulesDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/RulesDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/RulesDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/RulesDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/RulesInfoDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/RulesInfoDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/RulesInfoDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/RulesInfoDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/StateChangeTimeIntervalDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/StateChangeTimeIntervalDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/StateChangeTimeIntervalDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/StateChangeTimeIntervalDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/SubmissionTimeIntervalDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/SubmissionTimeIntervalDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/SubmissionTimeIntervalDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/SubmissionTimeIntervalDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/SubmitActionRequestDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/SubmitActionRequestDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/SubmitActionRequestDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/SubmitActionRequestDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/SubmitCmdletRequestDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/SubmitCmdletRequestDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/SubmitCmdletRequestDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/SubmitCmdletRequestDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/SubmitRuleRequestDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/SubmitRuleRequestDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/SubmitRuleRequestDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/SubmitRuleRequestDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/generated/model/UserInfoDto.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/UserInfoDto.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/generated/model/UserInfoDto.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/generated/model/UserInfoDto.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/mappers/ActionInfoMapper.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/mappers/ActionInfoMapper.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/mappers/ActionInfoMapper.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/mappers/ActionInfoMapper.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/mappers/AuditEventMapper.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/mappers/AuditEventMapper.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/mappers/AuditEventMapper.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/mappers/AuditEventMapper.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/mappers/CachedFileMapper.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/mappers/CachedFileMapper.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/mappers/CachedFileMapper.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/mappers/CachedFileMapper.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/mappers/ClusterNodeMapper.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/mappers/ClusterNodeMapper.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/mappers/ClusterNodeMapper.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/mappers/ClusterNodeMapper.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/mappers/CmdletInfoMapper.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/mappers/CmdletInfoMapper.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/mappers/CmdletInfoMapper.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/mappers/CmdletInfoMapper.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/mappers/FileAccessInfoMapper.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/mappers/FileAccessInfoMapper.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/mappers/FileAccessInfoMapper.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/mappers/FileAccessInfoMapper.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/mappers/RuleInfoMapper.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/mappers/RuleInfoMapper.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/mappers/RuleInfoMapper.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/mappers/RuleInfoMapper.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/mappers/SmartMapper.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/mappers/SmartMapper.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/mappers/SmartMapper.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/mappers/SmartMapper.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/mappers/pagination/ActionPageRequestMapper.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/mappers/pagination/ActionPageRequestMapper.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/mappers/pagination/ActionPageRequestMapper.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/mappers/pagination/ActionPageRequestMapper.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/mappers/pagination/AuditPageRequestMapper.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/mappers/pagination/AuditPageRequestMapper.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/mappers/pagination/AuditPageRequestMapper.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/mappers/pagination/AuditPageRequestMapper.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/mappers/pagination/BasePageRequestMapper.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/mappers/pagination/BasePageRequestMapper.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/mappers/pagination/BasePageRequestMapper.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/mappers/pagination/BasePageRequestMapper.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/mappers/pagination/CachedFilePageRequestMapper.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/mappers/pagination/CachedFilePageRequestMapper.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/mappers/pagination/CachedFilePageRequestMapper.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/mappers/pagination/CachedFilePageRequestMapper.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/mappers/pagination/ClusterNodePageRequestMapper.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/mappers/pagination/ClusterNodePageRequestMapper.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/mappers/pagination/ClusterNodePageRequestMapper.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/mappers/pagination/ClusterNodePageRequestMapper.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/mappers/pagination/CmdletsPageRequestMapper.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/mappers/pagination/CmdletsPageRequestMapper.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/mappers/pagination/CmdletsPageRequestMapper.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/mappers/pagination/CmdletsPageRequestMapper.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/mappers/pagination/FileAccessInfoPageRequestMapper.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/mappers/pagination/FileAccessInfoPageRequestMapper.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/mappers/pagination/FileAccessInfoPageRequestMapper.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/mappers/pagination/FileAccessInfoPageRequestMapper.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/mappers/pagination/RulesPageRequestMapper.java b/smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/mappers/pagination/RulesPageRequestMapper.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/mappers/pagination/RulesPageRequestMapper.java rename to smart-web-server/smart-master-web-server/src/main/java/org/smartdata/server/mappers/pagination/RulesPageRequestMapper.java diff --git a/smart-web-server/src/main/resources/api/parameters/actions/ActionSort.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/parameters/actions/ActionSort.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/parameters/actions/ActionSort.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/parameters/actions/ActionSort.yaml diff --git a/smart-web-server/src/main/resources/api/parameters/actions/action-sort.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/parameters/actions/action-sort.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/parameters/actions/action-sort.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/parameters/actions/action-sort.yaml diff --git a/smart-web-server/src/main/resources/api/parameters/actions/action-states.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/parameters/actions/action-states.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/parameters/actions/action-states.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/parameters/actions/action-states.yaml diff --git a/smart-web-server/src/main/resources/api/parameters/actions/completion-time.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/parameters/actions/completion-time.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/parameters/actions/completion-time.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/parameters/actions/completion-time.yaml diff --git a/smart-web-server/src/main/resources/api/parameters/actions/hosts.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/parameters/actions/hosts.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/parameters/actions/hosts.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/parameters/actions/hosts.yaml diff --git a/smart-web-server/src/main/resources/api/parameters/actions/sources.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/parameters/actions/sources.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/parameters/actions/sources.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/parameters/actions/sources.yaml diff --git a/smart-web-server/src/main/resources/api/parameters/audit/AuditSort.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/parameters/audit/AuditSort.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/parameters/audit/AuditSort.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/parameters/audit/AuditSort.yaml diff --git a/smart-web-server/src/main/resources/api/parameters/audit/audit-sort.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/parameters/audit/audit-sort.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/parameters/audit/audit-sort.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/parameters/audit/audit-sort.yaml diff --git a/smart-web-server/src/main/resources/api/parameters/audit/event-time.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/parameters/audit/event-time.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/parameters/audit/event-time.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/parameters/audit/event-time.yaml diff --git a/smart-web-server/src/main/resources/api/parameters/audit/object-ids.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/parameters/audit/object-ids.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/parameters/audit/object-ids.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/parameters/audit/object-ids.yaml diff --git a/smart-web-server/src/main/resources/api/parameters/audit/object-types.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/parameters/audit/object-types.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/parameters/audit/object-types.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/parameters/audit/object-types.yaml diff --git a/smart-web-server/src/main/resources/api/parameters/audit/operations.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/parameters/audit/operations.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/parameters/audit/operations.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/parameters/audit/operations.yaml diff --git a/smart-web-server/src/main/resources/api/parameters/audit/results.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/parameters/audit/results.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/parameters/audit/results.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/parameters/audit/results.yaml diff --git a/smart-web-server/src/main/resources/api/parameters/audit/username-like.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/parameters/audit/username-like.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/parameters/audit/username-like.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/parameters/audit/username-like.yaml diff --git a/smart-web-server/src/main/resources/api/parameters/cluster/ClusterSort.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/parameters/cluster/ClusterSort.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/parameters/cluster/ClusterSort.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/parameters/cluster/ClusterSort.yaml diff --git a/smart-web-server/src/main/resources/api/parameters/cluster/cluster-sort.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/parameters/cluster/cluster-sort.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/parameters/cluster/cluster-sort.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/parameters/cluster/cluster-sort.yaml diff --git a/smart-web-server/src/main/resources/api/parameters/cluster/registration-time.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/parameters/cluster/registration-time.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/parameters/cluster/registration-time.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/parameters/cluster/registration-time.yaml diff --git a/smart-web-server/src/main/resources/api/parameters/cmdlets/CmdletSort.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/parameters/cmdlets/CmdletSort.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/parameters/cmdlets/CmdletSort.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/parameters/cmdlets/CmdletSort.yaml diff --git a/smart-web-server/src/main/resources/api/parameters/cmdlets/cmdlets-sort.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/parameters/cmdlets/cmdlets-sort.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/parameters/cmdlets/cmdlets-sort.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/parameters/cmdlets/cmdlets-sort.yaml diff --git a/smart-web-server/src/main/resources/api/parameters/cmdlets/rule-ids.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/parameters/cmdlets/rule-ids.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/parameters/cmdlets/rule-ids.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/parameters/cmdlets/rule-ids.yaml diff --git a/smart-web-server/src/main/resources/api/parameters/cmdlets/state-changed-time.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/parameters/cmdlets/state-changed-time.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/parameters/cmdlets/state-changed-time.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/parameters/cmdlets/state-changed-time.yaml diff --git a/smart-web-server/src/main/resources/api/parameters/cmdlets/states.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/parameters/cmdlets/states.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/parameters/cmdlets/states.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/parameters/cmdlets/states.yaml diff --git a/smart-web-server/src/main/resources/api/parameters/common/PageRequest.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/parameters/common/PageRequest.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/parameters/common/PageRequest.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/parameters/common/PageRequest.yaml diff --git a/smart-web-server/src/main/resources/api/parameters/common/id.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/parameters/common/id.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/parameters/common/id.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/parameters/common/id.yaml diff --git a/smart-web-server/src/main/resources/api/parameters/common/page-request.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/parameters/common/page-request.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/parameters/common/page-request.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/parameters/common/page-request.yaml diff --git a/smart-web-server/src/main/resources/api/parameters/common/submission-time.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/parameters/common/submission-time.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/parameters/common/submission-time.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/parameters/common/submission-time.yaml diff --git a/smart-web-server/src/main/resources/api/parameters/common/text-representation-like.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/parameters/common/text-representation-like.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/parameters/common/text-representation-like.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/parameters/common/text-representation-like.yaml diff --git a/smart-web-server/src/main/resources/api/parameters/files/CachedFileSort.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/parameters/files/CachedFileSort.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/parameters/files/CachedFileSort.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/parameters/files/CachedFileSort.yaml diff --git a/smart-web-server/src/main/resources/api/parameters/files/HotFileSort.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/parameters/files/HotFileSort.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/parameters/files/HotFileSort.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/parameters/files/HotFileSort.yaml diff --git a/smart-web-server/src/main/resources/api/parameters/files/cached-files-sort.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/parameters/files/cached-files-sort.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/parameters/files/cached-files-sort.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/parameters/files/cached-files-sort.yaml diff --git a/smart-web-server/src/main/resources/api/parameters/files/cached-time.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/parameters/files/cached-time.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/parameters/files/cached-time.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/parameters/files/cached-time.yaml diff --git a/smart-web-server/src/main/resources/api/parameters/files/hot-files-sort.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/parameters/files/hot-files-sort.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/parameters/files/hot-files-sort.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/parameters/files/hot-files-sort.yaml diff --git a/smart-web-server/src/main/resources/api/parameters/files/last-accessed-time.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/parameters/files/last-accessed-time.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/parameters/files/last-accessed-time.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/parameters/files/last-accessed-time.yaml diff --git a/smart-web-server/src/main/resources/api/parameters/files/path-like.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/parameters/files/path-like.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/parameters/files/path-like.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/parameters/files/path-like.yaml diff --git a/smart-web-server/src/main/resources/api/parameters/rules/RuleSort.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/parameters/rules/RuleSort.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/parameters/rules/RuleSort.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/parameters/rules/RuleSort.yaml diff --git a/smart-web-server/src/main/resources/api/parameters/rules/last-activation-time.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/parameters/rules/last-activation-time.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/parameters/rules/last-activation-time.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/parameters/rules/last-activation-time.yaml diff --git a/smart-web-server/src/main/resources/api/parameters/rules/rule-states.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/parameters/rules/rule-states.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/parameters/rules/rule-states.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/parameters/rules/rule-states.yaml diff --git a/smart-web-server/src/main/resources/api/parameters/rules/rules-sort.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/parameters/rules/rules-sort.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/parameters/rules/rules-sort.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/parameters/rules/rules-sort.yaml diff --git a/smart-web-server/src/main/resources/api/resources/action.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/resources/action.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/resources/action.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/resources/action.yaml diff --git a/smart-web-server/src/main/resources/api/resources/actions.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/resources/actions.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/resources/actions.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/resources/actions.yaml diff --git a/smart-web-server/src/main/resources/api/resources/audit-events.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/resources/audit-events.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/resources/audit-events.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/resources/audit-events.yaml diff --git a/smart-web-server/src/main/resources/api/resources/cluster-nodes.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/resources/cluster-nodes.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/resources/cluster-nodes.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/resources/cluster-nodes.yaml diff --git a/smart-web-server/src/main/resources/api/resources/cmdlet-stop.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/resources/cmdlet-stop.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/resources/cmdlet-stop.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/resources/cmdlet-stop.yaml diff --git a/smart-web-server/src/main/resources/api/resources/cmdlet.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/resources/cmdlet.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/resources/cmdlet.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/resources/cmdlet.yaml diff --git a/smart-web-server/src/main/resources/api/resources/cmdlets.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/resources/cmdlets.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/resources/cmdlets.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/resources/cmdlets.yaml diff --git a/smart-web-server/src/main/resources/api/resources/current-user.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/resources/current-user.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/resources/current-user.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/resources/current-user.yaml diff --git a/smart-web-server/src/main/resources/api/resources/files-access-counts.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/resources/files-access-counts.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/resources/files-access-counts.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/resources/files-access-counts.yaml diff --git a/smart-web-server/src/main/resources/api/resources/files-cached.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/resources/files-cached.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/resources/files-cached.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/resources/files-cached.yaml diff --git a/smart-web-server/src/main/resources/api/resources/rule-start.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/resources/rule-start.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/resources/rule-start.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/resources/rule-start.yaml diff --git a/smart-web-server/src/main/resources/api/resources/rule-stop.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/resources/rule-stop.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/resources/rule-stop.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/resources/rule-stop.yaml diff --git a/smart-web-server/src/main/resources/api/resources/rule.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/resources/rule.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/resources/rule.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/resources/rule.yaml diff --git a/smart-web-server/src/main/resources/api/resources/rules-info.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/resources/rules-info.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/resources/rules-info.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/resources/rules-info.yaml diff --git a/smart-web-server/src/main/resources/api/resources/rules.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/resources/rules.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/resources/rules.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/resources/rules.yaml diff --git a/smart-web-server/src/main/resources/api/schemas/actions/Action.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/schemas/actions/Action.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/schemas/actions/Action.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/schemas/actions/Action.yaml diff --git a/smart-web-server/src/main/resources/api/schemas/actions/ActionInfo.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/schemas/actions/ActionInfo.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/schemas/actions/ActionInfo.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/schemas/actions/ActionInfo.yaml diff --git a/smart-web-server/src/main/resources/api/schemas/actions/ActionSource.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/schemas/actions/ActionSource.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/schemas/actions/ActionSource.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/schemas/actions/ActionSource.yaml diff --git a/smart-web-server/src/main/resources/api/schemas/actions/ActionState.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/schemas/actions/ActionState.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/schemas/actions/ActionState.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/schemas/actions/ActionState.yaml diff --git a/smart-web-server/src/main/resources/api/schemas/audit/AuditEvent.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/schemas/audit/AuditEvent.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/schemas/audit/AuditEvent.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/schemas/audit/AuditEvent.yaml diff --git a/smart-web-server/src/main/resources/api/schemas/audit/AuditEventResult.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/schemas/audit/AuditEventResult.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/schemas/audit/AuditEventResult.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/schemas/audit/AuditEventResult.yaml diff --git a/smart-web-server/src/main/resources/api/schemas/audit/AuditObjectType.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/schemas/audit/AuditObjectType.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/schemas/audit/AuditObjectType.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/schemas/audit/AuditObjectType.yaml diff --git a/smart-web-server/src/main/resources/api/schemas/audit/AuditOperation.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/schemas/audit/AuditOperation.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/schemas/audit/AuditOperation.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/schemas/audit/AuditOperation.yaml diff --git a/smart-web-server/src/main/resources/api/schemas/cluster/ClusterNode.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/schemas/cluster/ClusterNode.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/schemas/cluster/ClusterNode.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/schemas/cluster/ClusterNode.yaml diff --git a/smart-web-server/src/main/resources/api/schemas/cluster/ExecutorType.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/schemas/cluster/ExecutorType.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/schemas/cluster/ExecutorType.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/schemas/cluster/ExecutorType.yaml diff --git a/smart-web-server/src/main/resources/api/schemas/cmdlets/Cmdlet.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/schemas/cmdlets/Cmdlet.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/schemas/cmdlets/Cmdlet.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/schemas/cmdlets/Cmdlet.yaml diff --git a/smart-web-server/src/main/resources/api/schemas/cmdlets/CmdletState.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/schemas/cmdlets/CmdletState.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/schemas/cmdlets/CmdletState.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/schemas/cmdlets/CmdletState.yaml diff --git a/smart-web-server/src/main/resources/api/schemas/common/ErrorResponse.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/schemas/common/ErrorResponse.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/schemas/common/ErrorResponse.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/schemas/common/ErrorResponse.yaml diff --git a/smart-web-server/src/main/resources/api/schemas/common/PageResponse.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/schemas/common/PageResponse.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/schemas/common/PageResponse.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/schemas/common/PageResponse.yaml diff --git a/smart-web-server/src/main/resources/api/schemas/files/CachedFileInfo.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/schemas/files/CachedFileInfo.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/schemas/files/CachedFileInfo.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/schemas/files/CachedFileInfo.yaml diff --git a/smart-web-server/src/main/resources/api/schemas/files/FileAccessInfo.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/schemas/files/FileAccessInfo.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/schemas/files/FileAccessInfo.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/schemas/files/FileAccessInfo.yaml diff --git a/smart-web-server/src/main/resources/api/schemas/requests/SubmitActionRequest.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/schemas/requests/SubmitActionRequest.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/schemas/requests/SubmitActionRequest.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/schemas/requests/SubmitActionRequest.yaml diff --git a/smart-web-server/src/main/resources/api/schemas/requests/SubmitCmdletRequest.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/schemas/requests/SubmitCmdletRequest.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/schemas/requests/SubmitCmdletRequest.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/schemas/requests/SubmitCmdletRequest.yaml diff --git a/smart-web-server/src/main/resources/api/schemas/requests/SubmitRuleRequest.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/schemas/requests/SubmitRuleRequest.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/schemas/requests/SubmitRuleRequest.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/schemas/requests/SubmitRuleRequest.yaml diff --git a/smart-web-server/src/main/resources/api/schemas/rules/Rule.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/schemas/rules/Rule.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/schemas/rules/Rule.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/schemas/rules/Rule.yaml diff --git a/smart-web-server/src/main/resources/api/schemas/rules/RuleState.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/schemas/rules/RuleState.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/schemas/rules/RuleState.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/schemas/rules/RuleState.yaml diff --git a/smart-web-server/src/main/resources/api/schemas/rules/RulesInfo.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/schemas/rules/RulesInfo.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/schemas/rules/RulesInfo.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/schemas/rules/RulesInfo.yaml diff --git a/smart-web-server/src/main/resources/api/schemas/system/UserInfo.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/schemas/system/UserInfo.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/schemas/system/UserInfo.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/schemas/system/UserInfo.yaml diff --git a/smart-web-server/src/main/resources/api/ssm-api.yaml b/smart-web-server/smart-master-web-server/src/main/resources/api/ssm-api.yaml similarity index 100% rename from smart-web-server/src/main/resources/api/ssm-api.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/api/ssm-api.yaml diff --git a/smart-web-server/src/main/resources/application.yaml b/smart-web-server/smart-master-web-server/src/main/resources/application-master.yaml similarity index 81% rename from smart-web-server/src/main/resources/application.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/application-master.yaml index 10743fce0f3..3b64e3e148e 100644 --- a/smart-web-server/src/main/resources/application.yaml +++ b/smart-web-server/smart-master-web-server/src/main/resources/application-master.yaml @@ -9,6 +9,11 @@ server: session: cookie: name: SSM_SESSION_ID +management: + endpoints: + web: + exposure: + include: health,prometheus spring: web: resources: diff --git a/smart-web-server/src/main/resources/static/.openapi-generator/FILES b/smart-web-server/smart-master-web-server/src/main/resources/static/.openapi-generator/FILES similarity index 100% rename from smart-web-server/src/main/resources/static/.openapi-generator/FILES rename to smart-web-server/smart-master-web-server/src/main/resources/static/.openapi-generator/FILES diff --git a/smart-web-server/src/main/resources/static/.openapi-generator/VERSION b/smart-web-server/smart-master-web-server/src/main/resources/static/.openapi-generator/VERSION similarity index 100% rename from smart-web-server/src/main/resources/static/.openapi-generator/VERSION rename to smart-web-server/smart-master-web-server/src/main/resources/static/.openapi-generator/VERSION diff --git a/smart-web-server/src/main/resources/static/.openapi-generator/ssm-api.yaml-generate-merged-api-spec.sha256 b/smart-web-server/smart-master-web-server/src/main/resources/static/.openapi-generator/ssm-api.yaml-generate-merged-api-spec.sha256 similarity index 100% rename from smart-web-server/src/main/resources/static/.openapi-generator/ssm-api.yaml-generate-merged-api-spec.sha256 rename to smart-web-server/smart-master-web-server/src/main/resources/static/.openapi-generator/ssm-api.yaml-generate-merged-api-spec.sha256 diff --git a/smart-web-server/src/main/resources/static/ssm-api.yaml b/smart-web-server/smart-master-web-server/src/main/resources/static/ssm-api.yaml similarity index 100% rename from smart-web-server/src/main/resources/static/ssm-api.yaml rename to smart-web-server/smart-master-web-server/src/main/resources/static/ssm-api.yaml diff --git a/smart-web-server/smart-web-server-base/pom.xml b/smart-web-server/smart-web-server-base/pom.xml new file mode 100644 index 00000000000..adeddfe497c --- /dev/null +++ b/smart-web-server/smart-web-server-base/pom.xml @@ -0,0 +1,33 @@ + + + + 4.0.0 + + org.smartdata + smart-web-server + 2.0.0-SNAPSHOT + + + smart-web-server-base + + \ No newline at end of file diff --git a/smart-web-server/src/main/java/org/smartdata/server/SmartRestServer.java b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/SmartHttpServer.java similarity index 54% rename from smart-web-server/src/main/java/org/smartdata/server/SmartRestServer.java rename to smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/SmartHttpServer.java index c890f33df57..f9dbfb6d22f 100644 --- a/smart-web-server/src/main/java/org/smartdata/server/SmartRestServer.java +++ b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/SmartHttpServer.java @@ -15,13 +15,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server; +package org.smartdata.http; +import lombok.RequiredArgsConstructor; import org.smartdata.conf.SmartConf; -import org.smartdata.server.config.SsmContextInitializer; +import org.springframework.beans.factory.config.ConfigurableListableBeanFactory; import org.springframework.boot.SpringApplication; +import org.springframework.boot.actuate.autoconfigure.metrics.MetricsAutoConfiguration; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.hazelcast.HazelcastAutoConfiguration; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.context.ApplicationContextInitializer; import org.springframework.context.ConfigurableApplicationContext; import java.util.HashMap; @@ -29,22 +33,29 @@ import static org.smartdata.conf.SmartConfKeys.SMART_CONF_KEYS_PREFIX; -public class SmartRestServer { +public class SmartHttpServer { private static final String SPRING_LOGGING_PROPERTY = "org.springframework.boot.logging.LoggingSystem"; private static final String SPRING_DISABLED_LOGGER = "none"; + public static final String SERVER_PORT_QUALIFIER = "ssmHttpServerPort"; private final SpringApplication springApplication; private volatile ConfigurableApplicationContext applicationContext; - public SmartRestServer(SmartConf ssmConfig, SmartEngine smartEngine) { - this.springApplication = new SpringApplication(RestServerApplication.class); + public SmartHttpServer( + SmartConf ssmConfig, + ApplicationContextInitializer contextInitializer, + int serverPort, + Class... sources) { + SpringApplicationBuilder applicationBuilder = new SpringApplicationBuilder( + RestServerApplication.class) + .sources(sources) + .initializers(new BaseContextInitializer(ssmConfig, serverPort), contextInitializer) + .properties(getSsmProperties(ssmConfig)) + .profiles(); - injectToSpringProperties(ssmConfig); - SsmContextInitializer contextInitializer = - new SsmContextInitializer(smartEngine, ssmConfig); - springApplication.addInitializers(contextInitializer); + this.springApplication = customizeSpringApplication(applicationBuilder).build(); } public void start() { @@ -60,21 +71,43 @@ public void stop() { } } + protected SpringApplicationBuilder customizeSpringApplication( + SpringApplicationBuilder builder) { + return builder; + } + private boolean isRunning() { return applicationContext != null; } - private void injectToSpringProperties(SmartConf ssmConfig) { - Map ssmSpringProperties = new HashMap<>( + private Map getSsmProperties(SmartConf ssmConfig) { + return new HashMap<>( // we want to save only SSM-related options ssmConfig.asMap(key -> key.startsWith(SMART_CONF_KEYS_PREFIX))); - springApplication.setDefaultProperties(ssmSpringProperties); + } + + @RequiredArgsConstructor + private static class BaseContextInitializer implements + ApplicationContextInitializer { + + private final SmartConf conf; + private final int serverPort; + + @Override + public void initialize(ConfigurableApplicationContext applicationContext) { + ConfigurableListableBeanFactory beanFactory = applicationContext.getBeanFactory(); + beanFactory.registerSingleton(SERVER_PORT_QUALIFIER, serverPort); + beanFactory.registerSingleton("smartConfig", conf); + } } @SpringBootApplication(exclude = { // it's needed to prevent auto-registration of spring hazelcast node // in the SSM hazelcast workers cluster - HazelcastAutoConfiguration.class + HazelcastAutoConfiguration.class, + // we configure metrics registry by ourselves in + // MetricsFactory#from(SmartConf) before Spring context initialization + MetricsAutoConfiguration.class }) public static class RestServerApplication { // empty class just to enable auto configs diff --git a/smart-web-server/src/main/java/org/smartdata/server/config/ConfigKeys.java b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ConfigKeys.java similarity index 97% rename from smart-web-server/src/main/java/org/smartdata/server/config/ConfigKeys.java rename to smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ConfigKeys.java index b9283c726e4..6458df1fdb1 100644 --- a/smart-web-server/src/main/java/org/smartdata/server/config/ConfigKeys.java +++ b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ConfigKeys.java @@ -15,9 +15,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.config; +package org.smartdata.http.config; -import org.smartdata.server.config.ldap.search.LdapSearchScope; +import org.smartdata.http.config.ldap.search.LdapSearchScope; import java.util.Collections; import java.util.List; @@ -35,7 +35,7 @@ public class ConfigKeys { public static final String PREDEFINED_USERS = "smart.rest.server.auth.predefined.users"; - public static final String PREDEFINED_USERS_PASSWORD_ENCODER = + public static final String PREDEFINED_USERS_PASSWORD_ENCODER = "smart.rest.server.auth.predefined.password.encoder"; public static final String PREDEFINED_USERS_PASSWORD_ENCODER_DEFAULT = diff --git a/smart-web-server/src/main/java/org/smartdata/server/config/EmbeddedServerCustomizer.java b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/EmbeddedServerCustomizer.java similarity index 73% rename from smart-web-server/src/main/java/org/smartdata/server/config/EmbeddedServerCustomizer.java rename to smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/EmbeddedServerCustomizer.java index 347c2ad7e07..cb30791cb01 100644 --- a/smart-web-server/src/main/java/org/smartdata/server/config/EmbeddedServerCustomizer.java +++ b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/EmbeddedServerCustomizer.java @@ -15,10 +15,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.config; +package org.smartdata.http.config; import lombok.RequiredArgsConstructor; import org.smartdata.conf.SmartConf; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.web.server.ConfigurableWebServerFactory; import org.springframework.boot.web.server.Ssl; import org.springframework.boot.web.server.WebServerFactoryCustomizer; @@ -26,14 +27,13 @@ import java.util.Optional; -import static org.smartdata.conf.SmartConfKeys.SMART_REST_SERVER_PORT_KEY; -import static org.smartdata.conf.SmartConfKeys.SMART_REST_SERVER_PORT_KEY_DEFAULT; -import static org.smartdata.server.config.ConfigKeys.SSL_ENABLED; -import static org.smartdata.server.config.ConfigKeys.SSL_ENABLED_DEFAULT; -import static org.smartdata.server.config.ConfigKeys.SSL_KEYSTORE_PASSWORD; -import static org.smartdata.server.config.ConfigKeys.SSL_KEYSTORE_PATH; -import static org.smartdata.server.config.ConfigKeys.SSL_KEY_ALIAS; -import static org.smartdata.server.config.ConfigKeys.SSL_KEY_PASSWORD; +import static org.smartdata.http.SmartHttpServer.SERVER_PORT_QUALIFIER; +import static org.smartdata.http.config.ConfigKeys.SSL_ENABLED; +import static org.smartdata.http.config.ConfigKeys.SSL_ENABLED_DEFAULT; +import static org.smartdata.http.config.ConfigKeys.SSL_KEYSTORE_PASSWORD; +import static org.smartdata.http.config.ConfigKeys.SSL_KEYSTORE_PATH; +import static org.smartdata.http.config.ConfigKeys.SSL_KEY_ALIAS; +import static org.smartdata.http.config.ConfigKeys.SSL_KEY_PASSWORD; @Component @RequiredArgsConstructor @@ -42,10 +42,11 @@ public class EmbeddedServerCustomizer private final SmartConf conf; + @Qualifier(SERVER_PORT_QUALIFIER) + private final int serverPort; + @Override public void customize(ConfigurableWebServerFactory factory) { - int serverPort = conf.getInt(SMART_REST_SERVER_PORT_KEY, - SMART_REST_SERVER_PORT_KEY_DEFAULT); factory.setPort(serverPort); getSslConfig().ifPresent(factory::setSsl); } diff --git a/smart-web-server/src/main/java/org/smartdata/server/config/KerberosBasicAuthSecurityConfiguration.java b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/KerberosBasicAuthSecurityConfiguration.java similarity index 92% rename from smart-web-server/src/main/java/org/smartdata/server/config/KerberosBasicAuthSecurityConfiguration.java rename to smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/KerberosBasicAuthSecurityConfiguration.java index 2a494138e66..c1c270b76b8 100644 --- a/smart-web-server/src/main/java/org/smartdata/server/config/KerberosBasicAuthSecurityConfiguration.java +++ b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/KerberosBasicAuthSecurityConfiguration.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.config; +package org.smartdata.http.config; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; @@ -27,8 +27,8 @@ import org.springframework.security.kerberos.authentication.KerberosClient; import org.springframework.security.kerberos.authentication.sun.SunJaasKerberosClient; -import static org.smartdata.server.config.ConfigKeys.KERBEROS_BASIC_AUTH_ENABLED; -import static org.smartdata.server.config.ConfigKeys.WEB_SECURITY_ENABLED; +import static org.smartdata.http.config.ConfigKeys.KERBEROS_BASIC_AUTH_ENABLED; +import static org.smartdata.http.config.ConfigKeys.WEB_SECURITY_ENABLED; @Configuration @ConditionalOnProperty( diff --git a/smart-web-server/src/main/java/org/smartdata/server/config/LdapAuthSecurityConfiguration.java b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/LdapAuthSecurityConfiguration.java similarity index 79% rename from smart-web-server/src/main/java/org/smartdata/server/config/LdapAuthSecurityConfiguration.java rename to smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/LdapAuthSecurityConfiguration.java index 46d62867278..919005099ef 100644 --- a/smart-web-server/src/main/java/org/smartdata/server/config/LdapAuthSecurityConfiguration.java +++ b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/LdapAuthSecurityConfiguration.java @@ -15,11 +15,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.config; +package org.smartdata.http.config; import org.smartdata.conf.SmartConf; -import org.smartdata.server.config.PasswordEncoderFactory.EncoderType; -import org.smartdata.server.config.ldap.search.LdapUserSearchFactory; +import org.smartdata.http.config.PasswordEncoderFactory.EncoderType; +import org.smartdata.http.config.ldap.search.LdapUserSearchFactory; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -38,18 +38,18 @@ import java.util.Collection; import java.util.Optional; -import static org.smartdata.server.config.ConfigKeys.LDAP_AUTH_ENABLED; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_AUTH_TYPE; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_BIND_PASSWORD; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_BIND_USER_DN; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_PASSWORD_ENCODER; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_PASSWORD_ENCODER_DEFAULT; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_SEARCH_BASE; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_SEARCH_BASE_DEFAULT; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_URL; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_PASSWORD_ATTR; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_PASSWORD_ATTR_DEFAULT; -import static org.smartdata.server.config.ConfigKeys.WEB_SECURITY_ENABLED; +import static org.smartdata.http.config.ConfigKeys.LDAP_AUTH_ENABLED; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_AUTH_TYPE; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_BIND_PASSWORD; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_BIND_USER_DN; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_PASSWORD_ENCODER; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_PASSWORD_ENCODER_DEFAULT; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_SEARCH_BASE; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_SEARCH_BASE_DEFAULT; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_URL; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_PASSWORD_ATTR; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_PASSWORD_ATTR_DEFAULT; +import static org.smartdata.http.config.ConfigKeys.WEB_SECURITY_ENABLED; @Configuration @ConditionalOnProperty( diff --git a/smart-web-server/src/main/java/org/smartdata/server/config/PasswordEncoderFactory.java b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/PasswordEncoderFactory.java similarity index 98% rename from smart-web-server/src/main/java/org/smartdata/server/config/PasswordEncoderFactory.java rename to smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/PasswordEncoderFactory.java index fcda3810243..953e0aa98a3 100644 --- a/smart-web-server/src/main/java/org/smartdata/server/config/PasswordEncoderFactory.java +++ b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/PasswordEncoderFactory.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.config; +package org.smartdata.http.config; import lombok.Getter; import lombok.RequiredArgsConstructor; diff --git a/smart-web-server/src/main/java/org/smartdata/server/config/PredefinedUsersSecurityConfiguration.java b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/PredefinedUsersSecurityConfiguration.java similarity index 83% rename from smart-web-server/src/main/java/org/smartdata/server/config/PredefinedUsersSecurityConfiguration.java rename to smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/PredefinedUsersSecurityConfiguration.java index 8f585ae3836..9cec3bb50b3 100644 --- a/smart-web-server/src/main/java/org/smartdata/server/config/PredefinedUsersSecurityConfiguration.java +++ b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/PredefinedUsersSecurityConfiguration.java @@ -15,10 +15,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.config; +package org.smartdata.http.config; import org.smartdata.conf.SmartConf; -import org.smartdata.server.config.PasswordEncoderFactory.EncoderType; +import org.smartdata.http.config.PasswordEncoderFactory.EncoderType; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -30,11 +30,11 @@ import java.util.List; -import static org.smartdata.server.config.ConfigKeys.PREDEFINED_BASIC_AUTH_ENABLED; -import static org.smartdata.server.config.ConfigKeys.PREDEFINED_USERS_PASSWORD_ENCODER; -import static org.smartdata.server.config.ConfigKeys.PREDEFINED_USERS_PASSWORD_ENCODER_DEFAULT; -import static org.smartdata.server.config.ConfigKeys.WEB_SECURITY_ENABLED; -import static org.smartdata.server.util.ConfigUtils.parsePredefinedUsers; +import static org.smartdata.http.config.ConfigKeys.PREDEFINED_BASIC_AUTH_ENABLED; +import static org.smartdata.http.config.ConfigKeys.PREDEFINED_USERS_PASSWORD_ENCODER; +import static org.smartdata.http.config.ConfigKeys.PREDEFINED_USERS_PASSWORD_ENCODER_DEFAULT; +import static org.smartdata.http.config.ConfigKeys.WEB_SECURITY_ENABLED; +import static org.smartdata.http.util.ConfigUtils.parsePredefinedUsers; @Configuration @ConditionalOnProperty( diff --git a/smart-web-server/src/main/java/org/smartdata/server/config/SecurityConfiguration.java b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/SecurityConfiguration.java similarity index 90% rename from smart-web-server/src/main/java/org/smartdata/server/config/SecurityConfiguration.java rename to smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/SecurityConfiguration.java index 32473a5d4d2..295546722ae 100644 --- a/smart-web-server/src/main/java/org/smartdata/server/config/SecurityConfiguration.java +++ b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/SecurityConfiguration.java @@ -15,11 +15,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.config; +package org.smartdata.http.config; +import org.smartdata.http.security.SmartPrincipalInitializerFilter; import org.smartdata.security.SmartPrincipalManager; import org.smartdata.server.error.AuthenticationFailureListener; -import org.smartdata.server.security.SmartPrincipalInitializerFilter; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -34,13 +34,14 @@ import java.util.List; import java.util.Set; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_AUTH_ERRORS_LOGGING_ENABLED; -import static org.smartdata.server.config.ConfigKeys.WEB_SECURITY_ENABLED; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_AUTH_ERRORS_LOGGING_ENABLED; +import static org.smartdata.http.config.ConfigKeys.WEB_SECURITY_ENABLED; @Configuration public class SecurityConfiguration { private static final String SESSION_COOKIE_NAME = "SSM_SESSIONID"; private static final String API_ENDPOINTS_PATTERN = "/api/**"; + private static final String METRICS_ENDPOINT_PATTERN = "/actuator/prometheus"; @Bean @ConditionalOnProperty(name = WEB_SECURITY_ENABLED, havingValue = "true") @@ -70,7 +71,7 @@ public SecurityFilterChain securityFilterChain( Set authHttpConfigurers) throws Exception { baseHttpSecurity(http) .authorizeRequests() - .antMatchers(API_ENDPOINTS_PATTERN).authenticated() + .antMatchers(API_ENDPOINTS_PATTERN, METRICS_ENDPOINT_PATTERN).authenticated() .and() .anonymous().disable() .addFilterAfter( diff --git a/smart-web-server/src/main/java/org/smartdata/server/config/SpnegoSecurityConfiguration.java b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/SpnegoSecurityConfiguration.java similarity index 92% rename from smart-web-server/src/main/java/org/smartdata/server/config/SpnegoSecurityConfiguration.java rename to smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/SpnegoSecurityConfiguration.java index 2568b00de95..db2723f31b1 100644 --- a/smart-web-server/src/main/java/org/smartdata/server/config/SpnegoSecurityConfiguration.java +++ b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/SpnegoSecurityConfiguration.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.config; +package org.smartdata.http.config; import lombok.RequiredArgsConstructor; import org.apache.commons.lang3.StringUtils; @@ -38,10 +38,10 @@ import java.util.Optional; import static org.smartdata.conf.SmartConfKeys.SMART_SERVER_KEYTAB_FILE_KEY; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_KERBEROS_PRINCIPAL; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_KEYTAB_FILE_KEY; -import static org.smartdata.server.config.ConfigKeys.SPNEGO_AUTH_ENABLED; -import static org.smartdata.server.config.ConfigKeys.WEB_SECURITY_ENABLED; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_KERBEROS_PRINCIPAL; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_KEYTAB_FILE_KEY; +import static org.smartdata.http.config.ConfigKeys.SPNEGO_AUTH_ENABLED; +import static org.smartdata.http.config.ConfigKeys.WEB_SECURITY_ENABLED; @Configuration @ConditionalOnProperty( diff --git a/smart-web-server/src/main/java/org/smartdata/server/config/SsmAuthHttpConfigurer.java b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/SsmAuthHttpConfigurer.java similarity index 96% rename from smart-web-server/src/main/java/org/smartdata/server/config/SsmAuthHttpConfigurer.java rename to smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/SsmAuthHttpConfigurer.java index 18fb076f343..d7654590d27 100644 --- a/smart-web-server/src/main/java/org/smartdata/server/config/SsmAuthHttpConfigurer.java +++ b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/SsmAuthHttpConfigurer.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.config; +package org.smartdata.http.config; import org.springframework.security.config.annotation.web.builders.HttpSecurity; import org.springframework.security.config.annotation.web.configurers.AbstractHttpConfigurer; diff --git a/smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/AdditionalFilterTemplateFactoryWrapper.java b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/AdditionalFilterTemplateFactoryWrapper.java similarity index 86% rename from smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/AdditionalFilterTemplateFactoryWrapper.java rename to smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/AdditionalFilterTemplateFactoryWrapper.java index 7b9bc8d1c59..49ba064dca6 100644 --- a/smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/AdditionalFilterTemplateFactoryWrapper.java +++ b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/AdditionalFilterTemplateFactoryWrapper.java @@ -15,16 +15,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.config.ldap.search; +package org.smartdata.http.config.ldap.search; import org.smartdata.conf.SmartConf; -import org.smartdata.server.config.ldap.search.query.LdapExpressionTemplate; +import org.smartdata.http.config.ldap.search.query.LdapExpressionTemplate; import org.springframework.util.Assert; import java.util.Optional; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_SEARCH_ADDITIONAL_FILTER; -import static org.smartdata.server.config.ldap.search.query.LdapQueryDsl.and; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_SEARCH_ADDITIONAL_FILTER; +import static org.smartdata.http.config.ldap.search.query.LdapQueryDsl.and; public class AdditionalFilterTemplateFactoryWrapper implements LdapSearchTemplateFactory { diff --git a/smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/LdapSearchScope.java b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/LdapSearchScope.java similarity index 95% rename from smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/LdapSearchScope.java rename to smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/LdapSearchScope.java index 4feafbdb235..907e66d6d96 100644 --- a/smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/LdapSearchScope.java +++ b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/LdapSearchScope.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.config.ldap.search; +package org.smartdata.http.config.ldap.search; import lombok.Getter; import lombok.RequiredArgsConstructor; diff --git a/smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/LdapSearchTemplateFactory.java b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/LdapSearchTemplateFactory.java similarity index 87% rename from smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/LdapSearchTemplateFactory.java rename to smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/LdapSearchTemplateFactory.java index 50886d23ea8..55766d35558 100644 --- a/smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/LdapSearchTemplateFactory.java +++ b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/LdapSearchTemplateFactory.java @@ -15,9 +15,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.config.ldap.search; +package org.smartdata.http.config.ldap.search; -import org.smartdata.server.config.ldap.search.query.LdapExpressionTemplate; +import org.smartdata.http.config.ldap.search.query.LdapExpressionTemplate; public interface LdapSearchTemplateFactory { diff --git a/smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/LdapUserSearchFactory.java b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/LdapUserSearchFactory.java similarity index 77% rename from smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/LdapUserSearchFactory.java rename to smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/LdapUserSearchFactory.java index c23d31bf80f..70967b8b679 100644 --- a/smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/LdapUserSearchFactory.java +++ b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/LdapUserSearchFactory.java @@ -15,25 +15,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.config.ldap.search; +package org.smartdata.http.config.ldap.search; import lombok.extern.slf4j.Slf4j; import org.smartdata.conf.SmartConf; -import org.smartdata.server.config.ldap.search.group.GroupSearchByMemberAttrFactory; -import org.smartdata.server.config.ldap.search.group.GroupSearchRunner; -import org.smartdata.server.config.ldap.search.group.SsmLdapGroupSearch; -import org.smartdata.server.config.ldap.search.user.UserGroupSearchRunner; -import org.smartdata.server.config.ldap.search.user.UserSearchByCustomQueryFactory; -import org.smartdata.server.config.ldap.search.user.UserSearchByMembershipAttrFactory; -import org.smartdata.server.config.ldap.search.user.UserSearchByNameAttributeFactory; -import org.smartdata.server.config.ldap.search.user.UserSearchRunner; +import org.smartdata.http.config.ldap.search.group.GroupSearchByMemberAttrFactory; +import org.smartdata.http.config.ldap.search.group.GroupSearchRunner; +import org.smartdata.http.config.ldap.search.group.SsmLdapGroupSearch; +import org.smartdata.http.config.ldap.search.user.UserGroupSearchRunner; +import org.smartdata.http.config.ldap.search.user.UserSearchByCustomQueryFactory; +import org.smartdata.http.config.ldap.search.user.UserSearchByMembershipAttrFactory; +import org.smartdata.http.config.ldap.search.user.UserSearchByNameAttributeFactory; +import org.smartdata.http.config.ldap.search.user.UserSearchRunner; import org.springframework.ldap.core.support.BaseLdapPathContextSource; import org.springframework.security.ldap.search.LdapUserSearch; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_CUSTOM_SEARCH; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_GROUP_MEMBER_ATTR; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_GROUPS; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_MEMBER_ATTR; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_CUSTOM_SEARCH; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_GROUP_MEMBER_ATTR; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_GROUPS; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_MEMBER_ATTR; @Slf4j public class LdapUserSearchFactory { diff --git a/smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/LdapUtils.java b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/LdapUtils.java similarity index 88% rename from smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/LdapUtils.java rename to smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/LdapUtils.java index 7c50f78f9be..5f25e04d6b8 100644 --- a/smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/LdapUtils.java +++ b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/LdapUtils.java @@ -15,9 +15,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.config.ldap.search; +package org.smartdata.http.config.ldap.search; -import org.smartdata.server.config.ldap.search.query.LdapExpressionTemplate; +import org.smartdata.http.config.ldap.search.query.LdapExpressionTemplate; import org.springframework.ldap.support.LdapEncoder; import javax.naming.InvalidNameException; @@ -26,8 +26,8 @@ import java.text.MessageFormat; import java.util.Collection; -import static org.smartdata.server.config.ldap.search.query.LdapQueryDsl.eq; -import static org.smartdata.server.config.ldap.search.query.LdapQueryDsl.or; +import static org.smartdata.http.config.ldap.search.query.LdapQueryDsl.eq; +import static org.smartdata.http.config.ldap.search.query.LdapQueryDsl.or; import static org.springframework.ldap.support.LdapUtils.removeFirst; public class LdapUtils { diff --git a/smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/group/GroupSearchByMemberAttrFactory.java b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/group/GroupSearchByMemberAttrFactory.java similarity index 76% rename from smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/group/GroupSearchByMemberAttrFactory.java rename to smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/group/GroupSearchByMemberAttrFactory.java index fff60a383d5..682d4696188 100644 --- a/smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/group/GroupSearchByMemberAttrFactory.java +++ b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/group/GroupSearchByMemberAttrFactory.java @@ -15,18 +15,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.config.ldap.search.group; +package org.smartdata.http.config.ldap.search.group; import lombok.extern.slf4j.Slf4j; import org.smartdata.conf.SmartConf; -import org.smartdata.server.config.ldap.search.LdapSearchTemplateFactory; -import org.smartdata.server.config.ldap.search.query.LdapExpressionTemplate; +import org.smartdata.http.config.ldap.search.LdapSearchTemplateFactory; +import org.smartdata.http.config.ldap.search.query.LdapExpressionTemplate; import org.springframework.util.Assert; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_GROUP_MEMBER_ATTR; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_GROUP_MEMBER_DEFAULT; -import static org.smartdata.server.config.ldap.search.query.LdapQueryDsl.and; -import static org.smartdata.server.config.ldap.search.query.LdapQueryDsl.eq; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_GROUP_MEMBER_ATTR; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_GROUP_MEMBER_DEFAULT; +import static org.smartdata.http.config.ldap.search.query.LdapQueryDsl.and; +import static org.smartdata.http.config.ldap.search.query.LdapQueryDsl.eq; @Slf4j public class GroupSearchByMemberAttrFactory implements LdapSearchTemplateFactory { diff --git a/smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/group/GroupSearchByNameAttrFactory.java b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/group/GroupSearchByNameAttrFactory.java similarity index 69% rename from smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/group/GroupSearchByNameAttrFactory.java rename to smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/group/GroupSearchByNameAttrFactory.java index e203f7bd92f..b21b93c52d5 100644 --- a/smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/group/GroupSearchByNameAttrFactory.java +++ b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/group/GroupSearchByNameAttrFactory.java @@ -15,26 +15,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.config.ldap.search.group; +package org.smartdata.http.config.ldap.search.group; import lombok.extern.slf4j.Slf4j; import org.smartdata.conf.SmartConf; -import org.smartdata.server.config.ldap.search.LdapSearchTemplateFactory; -import org.smartdata.server.config.ldap.search.query.LdapExpressionTemplate; +import org.smartdata.http.config.ldap.search.LdapSearchTemplateFactory; +import org.smartdata.http.config.ldap.search.query.LdapExpressionTemplate; import org.springframework.util.Assert; import java.util.ArrayList; import java.util.List; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_GROUP_NAME_ATTR; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_GROUP_NAME_DEFAULT; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_GROUP_OBJECT; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_GROUP_OBJECT_DEFAULT; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_GROUPS; -import static org.smartdata.server.config.ldap.search.LdapUtils.isClassObject; -import static org.smartdata.server.config.ldap.search.query.LdapQueryDsl.and; -import static org.smartdata.server.config.ldap.search.query.LdapQueryDsl.eq; -import static org.smartdata.server.config.ldap.search.query.LdapQueryDsl.or; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_GROUP_NAME_ATTR; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_GROUP_NAME_DEFAULT; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_GROUP_OBJECT; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_GROUP_OBJECT_DEFAULT; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_GROUPS; +import static org.smartdata.http.config.ldap.search.LdapUtils.isClassObject; +import static org.smartdata.http.config.ldap.search.query.LdapQueryDsl.and; +import static org.smartdata.http.config.ldap.search.query.LdapQueryDsl.eq; +import static org.smartdata.http.config.ldap.search.query.LdapQueryDsl.or; @Slf4j public class GroupSearchByNameAttrFactory implements LdapSearchTemplateFactory { diff --git a/smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/group/GroupSearchRunner.java b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/group/GroupSearchRunner.java similarity index 81% rename from smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/group/GroupSearchRunner.java rename to smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/group/GroupSearchRunner.java index 17728ca4b23..e78289a4ba6 100644 --- a/smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/group/GroupSearchRunner.java +++ b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/group/GroupSearchRunner.java @@ -15,14 +15,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.config.ldap.search.group; +package org.smartdata.http.config.ldap.search.group; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.smartdata.conf.SmartConf; -import org.smartdata.server.config.ldap.search.LdapSearchScope; -import org.smartdata.server.config.ldap.search.LdapSearchTemplateFactory; -import org.smartdata.server.config.ldap.search.query.LdapExpressionTemplate; +import org.smartdata.http.config.ldap.search.LdapSearchScope; +import org.smartdata.http.config.ldap.search.LdapSearchTemplateFactory; +import org.smartdata.http.config.ldap.search.query.LdapExpressionTemplate; import org.springframework.ldap.core.ContextMapper; import org.springframework.ldap.core.DirContextAdapter; import org.springframework.ldap.core.support.BaseLdapPathContextSource; @@ -37,13 +37,13 @@ import java.util.List; import java.util.Optional; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_GROUP_SEARCH_BASE; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_GROUP_SEARCH_SCOPE; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_GROUP_SEARCH_SCOPE_DEFAULT; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_SEARCH_BASE; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_SEARCH_BASE_DEFAULT; -import static org.smartdata.server.config.ldap.search.LdapUtils.formatTemplate; -import static org.smartdata.server.config.ldap.search.LdapUtils.getRelativeBaseName; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_GROUP_SEARCH_BASE; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_GROUP_SEARCH_SCOPE; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_GROUP_SEARCH_SCOPE_DEFAULT; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_SEARCH_BASE; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_SEARCH_BASE_DEFAULT; +import static org.smartdata.http.config.ldap.search.LdapUtils.formatTemplate; +import static org.smartdata.http.config.ldap.search.LdapUtils.getRelativeBaseName; @Slf4j public class GroupSearchRunner implements SsmLdapGroupSearch { diff --git a/smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/group/SsmLdapGroupSearch.java b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/group/SsmLdapGroupSearch.java similarity index 88% rename from smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/group/SsmLdapGroupSearch.java rename to smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/group/SsmLdapGroupSearch.java index 129997ae5c6..022267aae4f 100644 --- a/smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/group/SsmLdapGroupSearch.java +++ b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/group/SsmLdapGroupSearch.java @@ -15,9 +15,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.config.ldap.search.group; +package org.smartdata.http.config.ldap.search.group; -import org.smartdata.server.config.ldap.search.query.LdapExpressionTemplate; +import org.smartdata.http.config.ldap.search.query.LdapExpressionTemplate; import javax.naming.Name; diff --git a/smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/query/LdapExpressionTemplate.java b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/query/LdapExpressionTemplate.java similarity index 95% rename from smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/query/LdapExpressionTemplate.java rename to smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/query/LdapExpressionTemplate.java index 7be09b7ad2e..3136aa37395 100644 --- a/smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/query/LdapExpressionTemplate.java +++ b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/query/LdapExpressionTemplate.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.config.ldap.search.query; +package org.smartdata.http.config.ldap.search.query; public interface LdapExpressionTemplate { LdapExpressionTemplate EMPTY_EXPRESSION = () -> ""; diff --git a/smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/query/LdapFilter.java b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/query/LdapFilter.java similarity index 95% rename from smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/query/LdapFilter.java rename to smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/query/LdapFilter.java index 0900eaeecbf..4ad6ccd2af7 100644 --- a/smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/query/LdapFilter.java +++ b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/query/LdapFilter.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.config.ldap.search.query; +package org.smartdata.http.config.ldap.search.query; import lombok.Data; diff --git a/smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/query/LdapOperator.java b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/query/LdapOperator.java similarity index 97% rename from smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/query/LdapOperator.java rename to smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/query/LdapOperator.java index c436dc8e61e..c93a5d61969 100644 --- a/smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/query/LdapOperator.java +++ b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/query/LdapOperator.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.config.ldap.search.query; +package org.smartdata.http.config.ldap.search.query; import lombok.Data; import lombok.RequiredArgsConstructor; diff --git a/smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/query/LdapQueryDsl.java b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/query/LdapQueryDsl.java similarity index 96% rename from smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/query/LdapQueryDsl.java rename to smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/query/LdapQueryDsl.java index 9553b7a763d..c69b2cb7e55 100644 --- a/smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/query/LdapQueryDsl.java +++ b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/query/LdapQueryDsl.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.config.ldap.search.query; +package org.smartdata.http.config.ldap.search.query; import java.util.List; diff --git a/smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/user/UserGroupSearchRunner.java b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/user/UserGroupSearchRunner.java similarity index 93% rename from smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/user/UserGroupSearchRunner.java rename to smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/user/UserGroupSearchRunner.java index fb8dd21db3b..cd69abe3f30 100644 --- a/smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/user/UserGroupSearchRunner.java +++ b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/user/UserGroupSearchRunner.java @@ -15,12 +15,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.config.ldap.search.user; +package org.smartdata.http.config.ldap.search.user; import lombok.extern.slf4j.Slf4j; import org.smartdata.conf.SmartConf; -import org.smartdata.server.config.ldap.search.LdapSearchTemplateFactory; -import org.smartdata.server.config.ldap.search.group.SsmLdapGroupSearch; +import org.smartdata.http.config.ldap.search.LdapSearchTemplateFactory; +import org.smartdata.http.config.ldap.search.group.SsmLdapGroupSearch; import org.springframework.ldap.core.DirContextOperations; import org.springframework.ldap.core.support.BaseLdapPathContextSource; import org.springframework.ldap.support.LdapUtils; diff --git a/smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/user/UserSearchByCustomQueryFactory.java b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/user/UserSearchByCustomQueryFactory.java similarity index 83% rename from smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/user/UserSearchByCustomQueryFactory.java rename to smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/user/UserSearchByCustomQueryFactory.java index 0d34f9f63b2..2bce30c4ca9 100644 --- a/smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/user/UserSearchByCustomQueryFactory.java +++ b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/user/UserSearchByCustomQueryFactory.java @@ -15,15 +15,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.config.ldap.search.user; +package org.smartdata.http.config.ldap.search.user; import lombok.extern.slf4j.Slf4j; import org.smartdata.conf.SmartConf; -import org.smartdata.server.config.ldap.search.LdapSearchTemplateFactory; -import org.smartdata.server.config.ldap.search.query.LdapExpressionTemplate; +import org.smartdata.http.config.ldap.search.LdapSearchTemplateFactory; +import org.smartdata.http.config.ldap.search.query.LdapExpressionTemplate; import org.springframework.util.Assert; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_CUSTOM_SEARCH; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_CUSTOM_SEARCH; @Slf4j public class UserSearchByCustomQueryFactory implements LdapSearchTemplateFactory { diff --git a/smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/user/UserSearchByMembershipAttrFactory.java b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/user/UserSearchByMembershipAttrFactory.java similarity index 79% rename from smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/user/UserSearchByMembershipAttrFactory.java rename to smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/user/UserSearchByMembershipAttrFactory.java index 1d4e80e1f86..3e631e2bf5a 100644 --- a/smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/user/UserSearchByMembershipAttrFactory.java +++ b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/user/UserSearchByMembershipAttrFactory.java @@ -15,23 +15,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.config.ldap.search.user; +package org.smartdata.http.config.ldap.search.user; import lombok.extern.slf4j.Slf4j; import org.smartdata.conf.SmartConf; -import org.smartdata.server.config.ldap.search.LdapSearchTemplateFactory; -import org.smartdata.server.config.ldap.search.group.SsmLdapGroupSearch; -import org.smartdata.server.config.ldap.search.query.LdapExpressionTemplate; +import org.smartdata.http.config.ldap.search.LdapSearchTemplateFactory; +import org.smartdata.http.config.ldap.search.group.SsmLdapGroupSearch; +import org.smartdata.http.config.ldap.search.query.LdapExpressionTemplate; import org.springframework.util.Assert; import java.util.Set; import java.util.stream.Collectors; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_MEMBER_ATTR; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_MEMBER_ATTR_DEFAULT; -import static org.smartdata.server.config.ldap.search.query.LdapQueryDsl.and; -import static org.smartdata.server.config.ldap.search.query.LdapQueryDsl.eq; -import static org.smartdata.server.config.ldap.search.query.LdapQueryDsl.or; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_MEMBER_ATTR; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_MEMBER_ATTR_DEFAULT; +import static org.smartdata.http.config.ldap.search.query.LdapQueryDsl.and; +import static org.smartdata.http.config.ldap.search.query.LdapQueryDsl.eq; +import static org.smartdata.http.config.ldap.search.query.LdapQueryDsl.or; @Slf4j public class UserSearchByMembershipAttrFactory implements LdapSearchTemplateFactory { diff --git a/smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/user/UserSearchByNameAttributeFactory.java b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/user/UserSearchByNameAttributeFactory.java similarity index 69% rename from smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/user/UserSearchByNameAttributeFactory.java rename to smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/user/UserSearchByNameAttributeFactory.java index 7fba9ca511b..d517c2636b7 100644 --- a/smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/user/UserSearchByNameAttributeFactory.java +++ b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/user/UserSearchByNameAttributeFactory.java @@ -15,23 +15,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.config.ldap.search.user; +package org.smartdata.http.config.ldap.search.user; import org.smartdata.conf.SmartConf; -import org.smartdata.server.config.ldap.search.LdapSearchTemplateFactory; -import org.smartdata.server.config.ldap.search.query.LdapExpressionTemplate; +import org.smartdata.http.config.ldap.search.LdapSearchTemplateFactory; +import org.smartdata.http.config.ldap.search.query.LdapExpressionTemplate; import org.springframework.util.Assert; import java.util.ArrayList; import java.util.List; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_NAME_ATTR; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_NAME_ATTR_DEFAULT; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_OBJECT; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_OBJECT_DEFAULT; -import static org.smartdata.server.config.ldap.search.LdapUtils.isClassObject; -import static org.smartdata.server.config.ldap.search.query.LdapQueryDsl.and; -import static org.smartdata.server.config.ldap.search.query.LdapQueryDsl.eq; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_NAME_ATTR; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_NAME_ATTR_DEFAULT; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_OBJECT; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_OBJECT_DEFAULT; +import static org.smartdata.http.config.ldap.search.LdapUtils.isClassObject; +import static org.smartdata.http.config.ldap.search.query.LdapQueryDsl.and; +import static org.smartdata.http.config.ldap.search.query.LdapQueryDsl.eq; public class UserSearchByNameAttributeFactory implements LdapSearchTemplateFactory { static final String USERNAME_INJECT_PATTERN = "{0}"; diff --git a/smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/user/UserSearchRunner.java b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/user/UserSearchRunner.java similarity index 84% rename from smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/user/UserSearchRunner.java rename to smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/user/UserSearchRunner.java index f9d04d4103e..abad6a19c13 100644 --- a/smart-web-server/src/main/java/org/smartdata/server/config/ldap/search/user/UserSearchRunner.java +++ b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/ldap/search/user/UserSearchRunner.java @@ -15,14 +15,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.config.ldap.search.user; +package org.smartdata.http.config.ldap.search.user; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.smartdata.conf.SmartConf; -import org.smartdata.server.config.ldap.search.AdditionalFilterTemplateFactoryWrapper; -import org.smartdata.server.config.ldap.search.LdapSearchScope; -import org.smartdata.server.config.ldap.search.LdapSearchTemplateFactory; +import org.smartdata.http.config.ldap.search.AdditionalFilterTemplateFactoryWrapper; +import org.smartdata.http.config.ldap.search.LdapSearchScope; +import org.smartdata.http.config.ldap.search.LdapSearchTemplateFactory; import org.springframework.dao.IncorrectResultSizeDataAccessException; import org.springframework.ldap.core.DirContextOperations; import org.springframework.ldap.core.support.BaseLdapPathContextSource; @@ -37,11 +37,11 @@ import java.util.Optional; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_SEARCH_BASE; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_SEARCH_BASE; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_SEARCH_SCOPE; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_SEARCH_SCOPE_DEFAULT; -import static org.smartdata.server.config.ldap.search.LdapUtils.getRelativeBaseName; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_SEARCH_BASE; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_SEARCH_BASE; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_SEARCH_SCOPE; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_SEARCH_SCOPE_DEFAULT; +import static org.smartdata.http.config.ldap.search.LdapUtils.getRelativeBaseName; @Slf4j public class UserSearchRunner implements LdapUserSearch { diff --git a/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/metrics/MetricsConfiguration.java b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/metrics/MetricsConfiguration.java new file mode 100644 index 00000000000..c12ed15859d --- /dev/null +++ b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/metrics/MetricsConfiguration.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.http.config.metrics; + +import io.micrometer.core.instrument.MeterRegistry; +import org.smartdata.metrics.MetricsFactory; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; + +import static org.smartdata.conf.SmartConfKeys.SMART_METRICS_ENABLED_KEY; + +@Configuration +@ConditionalOnProperty( + name = SMART_METRICS_ENABLED_KEY, + havingValue = "true", + matchIfMissing = true +) +public class MetricsConfiguration { + @Bean + @Primary + public MeterRegistry meterRegistry(MetricsFactory metricsFactory) { + return metricsFactory.getMeterRegistry(); + } +} diff --git a/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/metrics/PrometheusMetricsConfiguration.java b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/metrics/PrometheusMetricsConfiguration.java new file mode 100644 index 00000000000..7ec576d9163 --- /dev/null +++ b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/metrics/PrometheusMetricsConfiguration.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.http.config.metrics; + +import io.micrometer.prometheus.PrometheusMeterRegistry; +import io.prometheus.client.CollectorRegistry; +import org.smartdata.metrics.CompositeMetricsFactory; +import org.smartdata.metrics.MetricsFactory; +import org.springframework.boot.actuate.metrics.export.prometheus.PrometheusScrapeEndpoint; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import static org.smartdata.conf.SmartConfKeys.SMART_METRICS_ENABLED_KEY; +import static org.smartdata.conf.SmartConfKeys.SMART_METRICS_PROMETHEUS_ENABLED_KEY; + +@Configuration +@ConditionalOnProperty( + name = {SMART_METRICS_ENABLED_KEY, SMART_METRICS_PROMETHEUS_ENABLED_KEY}, + havingValue = "true", + matchIfMissing = true +) +public class PrometheusMetricsConfiguration { + + @Bean + public CollectorRegistry collectorRegistry(MetricsFactory metricsFactory) { + CompositeMetricsFactory compositeMetricsFactory = (CompositeMetricsFactory) metricsFactory; + return compositeMetricsFactory.getMeterRegistry() + .getRegistries() + .stream() + .filter(it -> it instanceof PrometheusMeterRegistry) + .map(PrometheusMeterRegistry.class::cast) + .map(PrometheusMeterRegistry::getPrometheusRegistry) + .findFirst() + .orElse(null); + } + + @Bean + public PrometheusScrapeEndpoint prometheusEndpoint(CollectorRegistry collectorRegistry) { + return new PrometheusScrapeEndpoint(collectorRegistry); + } +} diff --git a/smart-web-server/src/main/java/org/smartdata/server/error/AuthenticationFailureListener.java b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/error/AuthenticationFailureListener.java similarity index 100% rename from smart-web-server/src/main/java/org/smartdata/server/error/AuthenticationFailureListener.java rename to smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/error/AuthenticationFailureListener.java diff --git a/smart-web-server/src/main/java/org/smartdata/server/error/ErrorDto.java b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/error/ErrorDto.java similarity index 96% rename from smart-web-server/src/main/java/org/smartdata/server/error/ErrorDto.java rename to smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/error/ErrorDto.java index 2614ff3dffc..bdc64d13445 100644 --- a/smart-web-server/src/main/java/org/smartdata/server/error/ErrorDto.java +++ b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/error/ErrorDto.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.error; +package org.smartdata.http.error; import lombok.Getter; import lombok.RequiredArgsConstructor; diff --git a/smart-web-server/src/main/java/org/smartdata/server/error/SmartExceptionHandler.java b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/error/SmartExceptionHandler.java similarity index 86% rename from smart-web-server/src/main/java/org/smartdata/server/error/SmartExceptionHandler.java rename to smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/error/SmartExceptionHandler.java index 23ac4ce0f12..956c1f3a2e5 100644 --- a/smart-web-server/src/main/java/org/smartdata/server/error/SmartExceptionHandler.java +++ b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/error/SmartExceptionHandler.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.error; +package org.smartdata.http.error; import org.apache.commons.lang3.exception.ExceptionUtils; import org.slf4j.Logger; @@ -141,18 +141,18 @@ protected ResponseEntity handleBindException( HttpHeaders headers, HttpStatus status, WebRequest request) { - String errorMessage = exception.getBindingResult().getAllErrors().stream() - .map(DefaultMessageSourceResolvable::getDefaultMessage) - .collect(Collectors.joining(";")); - return handleExceptionInternal( - exception, - request, - HttpStatus.BAD_REQUEST, - new ErrorDto<>( - SsmErrorCode.VALIDATION_ERROR.getCode(), - errorMessage, - ExceptionUtils.getStackTrace(exception)) - ); + String errorMessage = exception.getBindingResult().getAllErrors().stream() + .map(DefaultMessageSourceResolvable::getDefaultMessage) + .collect(Collectors.joining(";")); + return handleExceptionInternal( + exception, + request, + HttpStatus.BAD_REQUEST, + new ErrorDto<>( + SsmErrorCode.VALIDATION_ERROR.getCode(), + errorMessage, + ExceptionUtils.getStackTrace(exception)) + ); } private ResponseEntity handleExceptionInternal( @@ -163,21 +163,21 @@ private ResponseEntity handleExceptionInternal( ExceptionUtils.getStackTrace(exception)); return handleExceptionInternal(exception, - request, - status, - errorBody); + request, + status, + errorBody); } private ResponseEntity handleExceptionInternal( - Exception exception, WebRequest request, HttpStatus status, ErrorDto errorBody) { + Exception exception, WebRequest request, HttpStatus status, ErrorDto errorBody) { LOG.error("Exception during handling request on {}", - request.getDescription(false), exception); + request.getDescription(false), exception); return handleExceptionInternal( - exception, - errorBody, - new HttpHeaders(), - status, - request); + exception, + errorBody, + new HttpHeaders(), + status, + request); } } diff --git a/smart-web-server/src/main/java/org/smartdata/server/error/SsmErrorCode.java b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/error/SsmErrorCode.java similarity index 96% rename from smart-web-server/src/main/java/org/smartdata/server/error/SsmErrorCode.java rename to smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/error/SsmErrorCode.java index 57dadb2f90c..2b09b3f8bfd 100644 --- a/smart-web-server/src/main/java/org/smartdata/server/error/SsmErrorCode.java +++ b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/error/SsmErrorCode.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.error; +package org.smartdata.http.error; import lombok.Getter; import lombok.RequiredArgsConstructor; diff --git a/smart-web-server/src/main/java/org/smartdata/server/security/SmartPrincipalInitializerFilter.java b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/security/SmartPrincipalInitializerFilter.java similarity index 98% rename from smart-web-server/src/main/java/org/smartdata/server/security/SmartPrincipalInitializerFilter.java rename to smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/security/SmartPrincipalInitializerFilter.java index 425a8efc465..08bc808557a 100644 --- a/smart-web-server/src/main/java/org/smartdata/server/security/SmartPrincipalInitializerFilter.java +++ b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/security/SmartPrincipalInitializerFilter.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.security; +package org.smartdata.http.security; import lombok.RequiredArgsConstructor; import org.smartdata.security.SmartPrincipal; diff --git a/smart-web-server/src/main/java/org/smartdata/server/util/ConfigUtils.java b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/util/ConfigUtils.java similarity index 95% rename from smart-web-server/src/main/java/org/smartdata/server/util/ConfigUtils.java rename to smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/util/ConfigUtils.java index 2a2f988e62c..a87a1fbbb25 100644 --- a/smart-web-server/src/main/java/org/smartdata/server/util/ConfigUtils.java +++ b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/util/ConfigUtils.java @@ -15,10 +15,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.util; +package org.smartdata.http.util; import org.smartdata.conf.SmartConf; -import org.smartdata.server.config.ConfigKeys; +import org.smartdata.http.config.ConfigKeys; import org.springframework.security.core.userdetails.User; import org.springframework.security.core.userdetails.UserDetails; diff --git a/smart-web-server/src/test/java/org/smartdata/server/config/ldap/search/TestAdditionalFilterTemplateFactoryWrapper.java b/smart-web-server/smart-web-server-base/src/test/java/org/smartdata/http/config/ldap/search/TestAdditionalFilterTemplateFactoryWrapper.java similarity index 82% rename from smart-web-server/src/test/java/org/smartdata/server/config/ldap/search/TestAdditionalFilterTemplateFactoryWrapper.java rename to smart-web-server/smart-web-server-base/src/test/java/org/smartdata/http/config/ldap/search/TestAdditionalFilterTemplateFactoryWrapper.java index baf720608cd..8cefa530948 100644 --- a/smart-web-server/src/test/java/org/smartdata/server/config/ldap/search/TestAdditionalFilterTemplateFactoryWrapper.java +++ b/smart-web-server/smart-web-server-base/src/test/java/org/smartdata/http/config/ldap/search/TestAdditionalFilterTemplateFactoryWrapper.java @@ -15,15 +15,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.config.ldap.search; +package org.smartdata.http.config.ldap.search; import org.junit.Test; import org.smartdata.conf.SmartConf; -import org.smartdata.server.config.ldap.search.user.UserSearchByNameAttributeFactory; +import org.smartdata.http.config.ldap.search.user.UserSearchByNameAttributeFactory; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_SEARCH_ADDITIONAL_FILTER; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_NAME_ATTR; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_OBJECT; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_SEARCH_ADDITIONAL_FILTER; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_NAME_ATTR; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_OBJECT; public class TestAdditionalFilterTemplateFactoryWrapper extends TestLdapSearchTemplateFactory { diff --git a/smart-web-server/src/test/java/org/smartdata/server/config/ldap/search/TestGroupSearchByMemberAttrFactory.java b/smart-web-server/smart-web-server-base/src/test/java/org/smartdata/http/config/ldap/search/TestGroupSearchByMemberAttrFactory.java similarity index 83% rename from smart-web-server/src/test/java/org/smartdata/server/config/ldap/search/TestGroupSearchByMemberAttrFactory.java rename to smart-web-server/smart-web-server-base/src/test/java/org/smartdata/http/config/ldap/search/TestGroupSearchByMemberAttrFactory.java index cb00d038411..3ca46de8cf8 100644 --- a/smart-web-server/src/test/java/org/smartdata/server/config/ldap/search/TestGroupSearchByMemberAttrFactory.java +++ b/smart-web-server/smart-web-server-base/src/test/java/org/smartdata/http/config/ldap/search/TestGroupSearchByMemberAttrFactory.java @@ -15,15 +15,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.config.ldap.search; +package org.smartdata.http.config.ldap.search; import org.junit.Test; import org.smartdata.conf.SmartConf; -import org.smartdata.server.config.ldap.search.group.GroupSearchByMemberAttrFactory; +import org.smartdata.http.config.ldap.search.group.GroupSearchByMemberAttrFactory; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_GROUP_MEMBER_ATTR; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_GROUP_NAME_ATTR; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_GROUPS; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_GROUP_MEMBER_ATTR; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_GROUP_NAME_ATTR; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_GROUPS; public class TestGroupSearchByMemberAttrFactory extends TestLdapSearchTemplateFactory { diff --git a/smart-web-server/src/test/java/org/smartdata/server/config/ldap/search/TestGroupSearchByNameAttrFactory.java b/smart-web-server/smart-web-server-base/src/test/java/org/smartdata/http/config/ldap/search/TestGroupSearchByNameAttrFactory.java similarity index 83% rename from smart-web-server/src/test/java/org/smartdata/server/config/ldap/search/TestGroupSearchByNameAttrFactory.java rename to smart-web-server/smart-web-server-base/src/test/java/org/smartdata/http/config/ldap/search/TestGroupSearchByNameAttrFactory.java index 9509a4d5c97..8811fe05516 100644 --- a/smart-web-server/src/test/java/org/smartdata/server/config/ldap/search/TestGroupSearchByNameAttrFactory.java +++ b/smart-web-server/smart-web-server-base/src/test/java/org/smartdata/http/config/ldap/search/TestGroupSearchByNameAttrFactory.java @@ -15,15 +15,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.config.ldap.search; +package org.smartdata.http.config.ldap.search; import org.junit.Test; import org.smartdata.conf.SmartConf; -import org.smartdata.server.config.ldap.search.group.GroupSearchByNameAttrFactory; +import org.smartdata.http.config.ldap.search.group.GroupSearchByNameAttrFactory; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_GROUP_NAME_ATTR; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_GROUP_OBJECT; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_GROUPS; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_GROUP_NAME_ATTR; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_GROUP_OBJECT; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_GROUPS; public class TestGroupSearchByNameAttrFactory extends TestLdapSearchTemplateFactory { diff --git a/smart-web-server/src/test/java/org/smartdata/server/config/ldap/search/TestLdapSearchTemplateFactory.java b/smart-web-server/smart-web-server-base/src/test/java/org/smartdata/http/config/ldap/search/TestLdapSearchTemplateFactory.java similarity index 96% rename from smart-web-server/src/test/java/org/smartdata/server/config/ldap/search/TestLdapSearchTemplateFactory.java rename to smart-web-server/smart-web-server-base/src/test/java/org/smartdata/http/config/ldap/search/TestLdapSearchTemplateFactory.java index 8bb84f93c9b..64da83e95c9 100644 --- a/smart-web-server/src/test/java/org/smartdata/server/config/ldap/search/TestLdapSearchTemplateFactory.java +++ b/smart-web-server/smart-web-server-base/src/test/java/org/smartdata/http/config/ldap/search/TestLdapSearchTemplateFactory.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.config.ldap.search; +package org.smartdata.http.config.ldap.search; import org.smartdata.conf.SmartConf; diff --git a/smart-web-server/src/test/java/org/smartdata/server/config/ldap/search/TestUserSearchByCustomQueryFactory.java b/smart-web-server/smart-web-server-base/src/test/java/org/smartdata/http/config/ldap/search/TestUserSearchByCustomQueryFactory.java similarity index 86% rename from smart-web-server/src/test/java/org/smartdata/server/config/ldap/search/TestUserSearchByCustomQueryFactory.java rename to smart-web-server/smart-web-server-base/src/test/java/org/smartdata/http/config/ldap/search/TestUserSearchByCustomQueryFactory.java index 5d2db78d447..83b7a1cd99f 100644 --- a/smart-web-server/src/test/java/org/smartdata/server/config/ldap/search/TestUserSearchByCustomQueryFactory.java +++ b/smart-web-server/smart-web-server-base/src/test/java/org/smartdata/http/config/ldap/search/TestUserSearchByCustomQueryFactory.java @@ -15,13 +15,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.config.ldap.search; +package org.smartdata.http.config.ldap.search; import org.junit.Test; import org.smartdata.conf.SmartConf; -import org.smartdata.server.config.ldap.search.user.UserSearchByCustomQueryFactory; +import org.smartdata.http.config.ldap.search.user.UserSearchByCustomQueryFactory; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_CUSTOM_SEARCH; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_CUSTOM_SEARCH; public class TestUserSearchByCustomQueryFactory extends TestLdapSearchTemplateFactory { diff --git a/smart-web-server/src/test/java/org/smartdata/server/config/ldap/search/TestUserSearchByMembershipAttrFactory.java b/smart-web-server/smart-web-server-base/src/test/java/org/smartdata/http/config/ldap/search/TestUserSearchByMembershipAttrFactory.java similarity index 83% rename from smart-web-server/src/test/java/org/smartdata/server/config/ldap/search/TestUserSearchByMembershipAttrFactory.java rename to smart-web-server/smart-web-server-base/src/test/java/org/smartdata/http/config/ldap/search/TestUserSearchByMembershipAttrFactory.java index f079ebe6437..7abcf36f348 100644 --- a/smart-web-server/src/test/java/org/smartdata/server/config/ldap/search/TestUserSearchByMembershipAttrFactory.java +++ b/smart-web-server/smart-web-server-base/src/test/java/org/smartdata/http/config/ldap/search/TestUserSearchByMembershipAttrFactory.java @@ -15,14 +15,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.config.ldap.search; +package org.smartdata.http.config.ldap.search; import lombok.RequiredArgsConstructor; import org.junit.Test; import org.smartdata.conf.SmartConf; -import org.smartdata.server.config.ldap.search.group.SsmLdapGroupSearch; -import org.smartdata.server.config.ldap.search.query.LdapExpressionTemplate; -import org.smartdata.server.config.ldap.search.user.UserSearchByMembershipAttrFactory; +import org.smartdata.http.config.ldap.search.group.SsmLdapGroupSearch; +import org.smartdata.http.config.ldap.search.query.LdapExpressionTemplate; +import org.smartdata.http.config.ldap.search.user.UserSearchByMembershipAttrFactory; import org.springframework.ldap.support.LdapUtils; import javax.naming.Name; @@ -32,9 +32,9 @@ import java.util.List; import java.util.stream.Collectors; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_GROUPS; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_MEMBER_ATTR; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_NAME_ATTR; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_GROUPS; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_MEMBER_ATTR; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_NAME_ATTR; public class TestUserSearchByMembershipAttrFactory extends TestLdapSearchTemplateFactory { diff --git a/smart-web-server/src/test/java/org/smartdata/server/config/ldap/search/TestUserSearchByNameAttrFactory.java b/smart-web-server/smart-web-server-base/src/test/java/org/smartdata/http/config/ldap/search/TestUserSearchByNameAttrFactory.java similarity index 84% rename from smart-web-server/src/test/java/org/smartdata/server/config/ldap/search/TestUserSearchByNameAttrFactory.java rename to smart-web-server/smart-web-server-base/src/test/java/org/smartdata/http/config/ldap/search/TestUserSearchByNameAttrFactory.java index 9e9f2db3c28..fea2fa06f87 100644 --- a/smart-web-server/src/test/java/org/smartdata/server/config/ldap/search/TestUserSearchByNameAttrFactory.java +++ b/smart-web-server/smart-web-server-base/src/test/java/org/smartdata/http/config/ldap/search/TestUserSearchByNameAttrFactory.java @@ -15,14 +15,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.config.ldap.search; +package org.smartdata.http.config.ldap.search; import org.junit.Test; import org.smartdata.conf.SmartConf; -import org.smartdata.server.config.ldap.search.user.UserSearchByNameAttributeFactory; +import org.smartdata.http.config.ldap.search.user.UserSearchByNameAttributeFactory; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_NAME_ATTR; -import static org.smartdata.server.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_OBJECT; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_NAME_ATTR; +import static org.smartdata.http.config.ConfigKeys.SMART_REST_SERVER_LDAP_USER_OBJECT; public class TestUserSearchByNameAttrFactory extends TestLdapSearchTemplateFactory { diff --git a/smart-web-server/src/test/java/org/smartdata/server/util/ConfigUtilsTest.java b/smart-web-server/smart-web-server-base/src/test/java/org/smartdata/http/util/ConfigUtilsTest.java similarity index 94% rename from smart-web-server/src/test/java/org/smartdata/server/util/ConfigUtilsTest.java rename to smart-web-server/smart-web-server-base/src/test/java/org/smartdata/http/util/ConfigUtilsTest.java index 29b0a5e66e6..2398fbff994 100644 --- a/smart-web-server/src/test/java/org/smartdata/server/util/ConfigUtilsTest.java +++ b/smart-web-server/smart-web-server-base/src/test/java/org/smartdata/http/util/ConfigUtilsTest.java @@ -15,11 +15,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.smartdata.server.util; +package org.smartdata.http.util; import org.junit.Test; import org.smartdata.conf.SmartConf; -import org.smartdata.server.config.ConfigKeys; +import org.smartdata.http.config.ConfigKeys; import org.springframework.security.core.userdetails.User; import org.springframework.security.core.userdetails.UserDetails; @@ -29,7 +29,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.smartdata.server.util.ConfigUtils.parsePredefinedUsers; +import static org.smartdata.http.util.ConfigUtils.parsePredefinedUsers; public class ConfigUtilsTest { diff --git a/supports/tools/docker/multihost/conf/prometheus.yml b/supports/tools/docker/multihost/conf/prometheus.yml new file mode 100644 index 00000000000..9b98e560797 --- /dev/null +++ b/supports/tools/docker/multihost/conf/prometheus.yml @@ -0,0 +1,18 @@ +global: + scrape_interval: 5s + +scrape_configs: + - job_name: 'ssm-server' + metrics_path: '/actuator/prometheus' + static_configs: + - targets: ['ssm-server.demo:8081'] + basic_auth: + username: 'john' + password: '1234' + - job_name: 'ssm-agent' + metrics_path: '/actuator/prometheus' + static_configs: + - targets: [ 'hadoop-datanode.demo:8081' ] + basic_auth: + username: 'john' + password: '1234' diff --git a/supports/tools/docker/multihost/conf/smart-site.xml b/supports/tools/docker/multihost/conf/smart-site.xml index f834968d313..b3711db0af9 100644 --- a/supports/tools/docker/multihost/conf/smart-site.xml +++ b/supports/tools/docker/multihost/conf/smart-site.xml @@ -89,7 +89,7 @@ smart.rest.server.auth.spnego.principal - HTTP/ssm-server.demo@DEMO + HTTP/_HOST@DEMO smart.rest.server.auth.ldap.enabled diff --git a/supports/tools/docker/multihost/docker-compose.yaml b/supports/tools/docker/multihost/docker-compose.yaml index 8f03cc63e08..83396fff013 100644 --- a/supports/tools/docker/multihost/docker-compose.yaml +++ b/supports/tools/docker/multihost/docker-compose.yaml @@ -124,6 +124,18 @@ services: networks: - demo + prometheus: + image: prom/prometheus:latest + container_name: prometheus + volumes: + - ./conf/prometheus.yml:/etc/prometheus/prometheus.yml + ports: + - "9090:9090" + command: + - '--config.file=/etc/prometheus/prometheus.yml' + networks: + - demo + networks: demo: name: demo diff --git a/supports/tools/docker/multihost/kerberos/kdc-init.sh b/supports/tools/docker/multihost/kerberos/kdc-init.sh index 19a01a34c21..54fe099988a 100644 --- a/supports/tools/docker/multihost/kerberos/kdc-init.sh +++ b/supports/tools/docker/multihost/kerberos/kdc-init.sh @@ -26,6 +26,7 @@ echo "==== Creating SSM, HDFS and YARN principals ==========" kadmin.local -q "add_principal -randkey namenode/hadoop-namenode.demo@DEMO" kadmin.local -q "add_principal -randkey datanode/hadoop-datanode.demo@DEMO" kadmin.local -q "add_principal -randkey HTTP/ssm-server.demo@DEMO" +kadmin.local -q "add_principal -randkey HTTP/hadoop-datanode.demo@DEMO" kadmin.local -q "add_principal -randkey ssm/ssm-server.demo@DEMO" kadmin.local -q "add_principal -randkey agent/hadoop-datanode.demo@DEMO" kadmin.local -q "add_principal -randkey agent/ssm-server.demo@DEMO" @@ -42,6 +43,7 @@ echo "==== Export keytabs for SSM, HDFS and YARN ==========" kadmin.local -q "xst -kt /tmp/secrets/namenode.keytab namenode/hadoop-namenode.demo@DEMO" && chown appuser:appuser /tmp/secrets/namenode.keytab kadmin.local -q "xst -kt /tmp/secrets/datanode.keytab datanode/hadoop-datanode.demo@DEMO" && chown appuser:appuser /tmp/secrets/datanode.keytab kadmin.local -q "xst -kt /tmp/secrets/http.keytab HTTP/ssm-server.demo@DEMO" && chown appuser:appuser /tmp/secrets/http.keytab +kadmin.local -q "xst -kt /tmp/secrets/http.keytab HTTP/hadoop-datanode.demo@DEMO" && chown appuser:appuser /tmp/secrets/http.keytab kadmin.local -q "xst -kt /tmp/secrets/ssm.keytab ssm/ssm-server.demo@DEMO" && chown appuser:appuser /tmp/secrets/ssm.keytab kadmin.local -q "xst -kt /tmp/secrets/agent.keytab agent/hadoop-datanode.demo@DEMO" && chown appuser:appuser /tmp/secrets/agent.keytab kadmin.local -q "xst -kt /tmp/secrets/agent.keytab agent/ssm-server.demo@DEMO" && chown appuser:appuser /tmp/secrets/agent.keytab From 8bf8d6cc953e9e91201f78863f1056177907edd1 Mon Sep 17 00:00:00 2001 From: Tigran Manasyan Date: Mon, 7 Oct 2024 19:30:41 +0400 Subject: [PATCH 2/3] [ADH-5129] Add RPC server metrics --- smart-metrics/README.md | 144 ++++++++++-------- .../org/smartdata/server/SmartRpcServer.java | 29 +++- .../SmartRpcServerMethodMetricsBinder.java | 102 +++++++++++++ .../server/SmartRpcServerMetricsBinder.java | 113 ++++++++++++++ .../org/smartdata/server/SmartServer.java | 3 +- .../resources/hadoop-metrics2-ssm.properties | 3 +- .../http/config/SecurityConfiguration.java | 2 +- 7 files changed, 322 insertions(+), 74 deletions(-) create mode 100644 smart-server/src/main/java/org/smartdata/server/SmartRpcServerMethodMetricsBinder.java create mode 100644 smart-server/src/main/java/org/smartdata/server/SmartRpcServerMetricsBinder.java diff --git a/smart-metrics/README.md b/smart-metrics/README.md index 47ca6dc63cb..9826a41dd39 100644 --- a/smart-metrics/README.md +++ b/smart-metrics/README.md @@ -1,69 +1,79 @@ # Supported metrics -| Name | Description | -|-----------------------------------------------------|---------------------------------------------------------------------------------------------------------| -| access_events_count_total | Access events count | -| application_ready_time_seconds | Time taken for the application to be ready to service requests | -| application_started_time_seconds | Time taken to start the application | -| db_pool_druid_connections_active_count | The number of active connections in the connection pool | -| db_pool_druid_connections_commit_count_total | The number of commits called | -| db_pool_druid_connections_connected_total | The number of opened connections in the connection pool | -| db_pool_druid_connections_created_total | The number of created connections in the connection pool | -| db_pool_druid_connections_error_connect_count_total | The number of connection errors during query executions | -| db_pool_druid_connections_error_count_total | The number of errors during query executions | -| db_pool_druid_connections_pooling_count | The number of idle connections in the connection pool | -| db_pool_druid_connections_rollback_count_total | The number of rollbacks called | -| db_pool_druid_lock_wait_queue_size | The size of threads waiting for connection from pool | -| db_pool_druid_lock_wait_time_ms | The total time of threads waiting for connection from pool | -| db_pool_druid_statements_execute_count_total | The number of prepared statements executions | -| db_pool_druid_statements_prepare_count_total | The number of prepared statements prepare method calls | -| db_pool_druid_statements_running_count | The number of running prepared statements | -| executor_active_threads | The approximate number of threads that are actively executing tasks | -| executor_completed_tasks_total | The approximate total number of tasks that have completed execution | -| executor_idle_seconds | Executor idle time | -| executor_idle_seconds_max | Maximum idle time for executor | -| executor_pool_core_threads | The core number of threads for the pool | -| executor_pool_max_threads | The maximum allowed number of threads in the pool | -| executor_pool_size_threads | The current number of threads in the pool | -| executor_queue_remaining_tasks | The number of additional elements that this queue can ideally accept without blocking | -| executor_queued_tasks | The approximate number of tasks that are queued for execution | -| executor_scheduled_once_total | The total number of tasks scheduled once | -| executor_scheduled_repetitively_total | The total number of tasks scheduled repetitively | -| executor_seconds | Total time spent by executor | -| executor_seconds_max | Maximum time spent by executor | -| http_server_requests_seconds | Duration of HTTP server request handling | -| http_server_requests_seconds_max | Maximum duration of HTTP server request handling | -| jvm_buffer_count_buffers | An estimate of the number of buffers in the pool | -| jvm_buffer_memory_used_bytes | An estimate of the memory that the Java virtual machine is using for this buffer pool | -| jvm_buffer_total_capacity_bytes | An estimate of the total capacity of the buffers in this pool | -| jvm_classes_loaded_classes | The number of classes that are currently loaded in the Java virtual machine | -| jvm_classes_unloaded_classes_total | The total number of classes unloaded since the Java virtual machine has started execution | -| jvm_gc_live_data_size_bytes | Size of long-lived heap memory pool after reclamation | -| jvm_gc_max_data_size_bytes | Max size of long-lived heap memory pool | -| jvm_gc_memory_allocated_bytes_total | Incremented for an increase in the size of the (young) heap memory pool after one GC to before the next | -| jvm_gc_memory_promoted_bytes_total | Count of positive increases in the size of the old generation memory pool before GC to after GC | -| jvm_gc_overhead_percent | Percent of CPU time used by GC activities over the last lookback period or since monitoring began | -| jvm_gc_pause_seconds | Time spent in GC pause | -| jvm_gc_pause_seconds_max | Maximum time spent in GC pause | -| jvm_memory_committed_bytes | The amount of memory in bytes that is committed for the Java virtual machine to use | -| jvm_memory_max_bytes | The maximum amount of memory in bytes that can be used for memory management | -| jvm_memory_usage_after_gc_percent | The percentage of long-lived heap pool used after the last GC event, in the range [0..1] | -| jvm_memory_used_bytes | The amount of used memory | -| jvm_threads_daemon_threads | The current number of live daemon threads | -| jvm_threads_live_threads | The current number of live threads including both daemon and non-daemon threads | -| jvm_threads_peak_threads | The peak live thread count since the Java virtual machine started or peak was reset | -| jvm_threads_states_threads | The current number of threads in each state | -| process_cpu_usage | The recent CPU usage for the Java Virtual Machine process | -| process_files_max_files | The maximum file descriptor count | -| process_files_open_files | The open file descriptor count | -| process_start_time_seconds | Start time of the process since Unix epoch | -| process_uptime_seconds | The uptime of the Java virtual machine | -| system_cpu_count | The number of processors available to the Java virtual machine | -| system_cpu_usage | The recent CPU usage of the system the application is running in | -| system_load_average_1m | The sum of runnable entities queued and running on available processors averaged over a period of time | -| tomcat_sessions_active_current_sessions | The number of current active sessions | -| tomcat_sessions_active_max_sessions | The maximum number of active sessions | -| tomcat_sessions_alive_max_seconds | The maximum number of seconds a session was alive | -| tomcat_sessions_created_sessions_total | The total number of sessions created | -| tomcat_sessions_expired_sessions_total | The total number of expired sessions | -| tomcat_sessions_rejected_sessions_total | The total number of rejected sessions | +| Name | Description | +|-----------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------| +| access_events_count_total | Access events count | +| application_ready_time_seconds | Time taken for the application to be ready to service requests | +| application_started_time_seconds | Time taken to start the application | +| db_pool_druid_connections_active_count | The number of active connections in the connection pool | +| db_pool_druid_connections_commit_count_total | The number of commits called | +| db_pool_druid_connections_connected_total | The number of opened connections in the connection pool | +| db_pool_druid_connections_created_total | The number of created connections in the connection pool | +| db_pool_druid_connections_error_connect_count_total | The number of connection errors during query executions | +| db_pool_druid_connections_error_count_total | The number of errors during query executions | +| db_pool_druid_connections_pooling_count | The number of idle connections in the connection pool | +| db_pool_druid_connections_rollback_count_total | The number of rollbacks called | +| db_pool_druid_lock_wait_queue_size | The size of threads waiting for connection from pool | +| db_pool_druid_lock_wait_time_ms | The total time of threads waiting for connection from pool | +| db_pool_druid_statements_execute_count_total | The number of prepared statements executions | +| db_pool_druid_statements_prepare_count_total | The number of prepared statements prepare method calls | +| db_pool_druid_statements_running_count | The number of running prepared statements | +| executor_active_threads | The approximate number of threads that are actively executing tasks | +| executor_completed_tasks_total | The approximate total number of tasks that have completed execution | +| executor_idle_seconds | Executor idle time | +| executor_idle_seconds_max | Maximum idle time for executor | +| executor_pool_core_threads | The core number of threads for the pool | +| executor_pool_max_threads | The maximum allowed number of threads in the pool | +| executor_pool_size_threads | The current number of threads in the pool | +| executor_queue_remaining_tasks | The number of additional elements that this queue can ideally accept without blocking | +| executor_queued_tasks | The approximate number of tasks that are queued for execution | +| executor_scheduled_once_total | The total number of tasks scheduled once | +| executor_scheduled_repetitively_total | The total number of tasks scheduled repetitively | +| executor_seconds | Total time spent by executor | +| executor_seconds_max | Maximum time spent by executor | +| http_server_requests_seconds | Duration of HTTP server request handling. Contains HTTP method name in the tag "method", status code in the tag "status" and path in the tag "uri" | +| http_server_requests_seconds_max | Maximum duration of HTTP server request handling | +| jvm_buffer_count_buffers | An estimate of the number of buffers in the pool | +| jvm_buffer_memory_used_bytes | An estimate of the memory that the Java virtual machine is using for this buffer pool | +| jvm_buffer_total_capacity_bytes | An estimate of the total capacity of the buffers in this pool | +| jvm_classes_loaded_classes | The number of classes that are currently loaded in the Java virtual machine | +| jvm_classes_unloaded_classes_total | The total number of classes unloaded since the Java virtual machine has started execution | +| jvm_gc_live_data_size_bytes | Size of long-lived heap memory pool after reclamation | +| jvm_gc_max_data_size_bytes | Max size of long-lived heap memory pool | +| jvm_gc_memory_allocated_bytes_total | Incremented for an increase in the size of the (young) heap memory pool after one GC to before the next | +| jvm_gc_memory_promoted_bytes_total | Count of positive increases in the size of the old generation memory pool before GC to after GC | +| jvm_gc_overhead_percent | Percent of CPU time used by GC activities over the last lookback period or since monitoring began | +| jvm_gc_pause_seconds | Time spent in GC pause | +| jvm_gc_pause_seconds_max | Maximum time spent in GC pause | +| jvm_memory_committed_bytes | The amount of memory in bytes that is committed for the Java virtual machine to use | +| jvm_memory_max_bytes | The maximum amount of memory in bytes that can be used for memory management | +| jvm_memory_usage_after_gc_percent | The percentage of long-lived heap pool used after the last GC event, in the range [0..1] | +| jvm_memory_used_bytes | The amount of used memory | +| jvm_threads_daemon_threads | The current number of live daemon threads | +| jvm_threads_live_threads | The current number of live threads including both daemon and non-daemon threads | +| jvm_threads_peak_threads | The peak live thread count since the Java virtual machine started or peak was reset | +| jvm_threads_states_threads | The current number of threads in each state | +| process_cpu_usage | The recent CPU usage for the Java Virtual Machine process | +| process_files_max_files | The maximum file descriptor count | +| process_files_open_files | The open file descriptor count | +| process_start_time_seconds | Start time of the process since Unix epoch | +| process_uptime_seconds | The uptime of the Java virtual machine | +| system_cpu_count | The number of processors available to the Java virtual machine | +| system_cpu_usage | The recent CPU usage of the system the application is running in | +| system_load_average_1m | The sum of runnable entities queued and running on available processors averaged over a period of time | +| tomcat_sessions_active_current_sessions | The number of current active sessions | +| tomcat_sessions_active_max_sessions | The maximum number of active sessions | +| tomcat_sessions_alive_max_seconds | The maximum number of seconds a session was alive | +| tomcat_sessions_created_sessions_total | The total number of sessions created | +| tomcat_sessions_expired_sessions_total | The total number of expired sessions | +| tomcat_sessions_rejected_sessions_total | The total number of rejected sessions | +| rpc_bytes_received | Total number of received bytes by RPC server | +| rpc_bytes_sent | Total number of sent bytes by RPC server | +| rpc_calls_total | Total number of RPC calls | +| rpc_requests_served_total | Total num of requests served by the RPC server | +| rpc_connections_open | Current number of open RPC connections | +| rpc_call_queue_size | Current length of the RPC call queue | +| rpc_call_authentication_success_total | Total number of RPC authentication successes | +| rpc_call_authentication_failure_total | Total number of RPC authentication failures | +| rpc_method_calls_total | Total number of the times the RPC method is called. Contains RPC method name in the tag "method" | +| rpc_method_call_time_average_ms | Average turn around time of the RPC method in milliseconds. Contains RPC method name in the tag "method" | diff --git a/smart-server/src/main/java/org/smartdata/server/SmartRpcServer.java b/smart-server/src/main/java/org/smartdata/server/SmartRpcServer.java index 57b0ad155c8..25a935e952f 100644 --- a/smart-server/src/main/java/org/smartdata/server/SmartRpcServer.java +++ b/smart-server/src/main/java/org/smartdata/server/SmartRpcServer.java @@ -23,11 +23,14 @@ import org.apache.hadoop.ipc.ProtobufRpcEngine2; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.RetriableException; +import org.apache.hadoop.metrics2.MetricsSystem; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.thirdparty.protobuf.BlockingService; import org.smartdata.SmartPolicyProvider; import org.smartdata.conf.SmartConfKeys; import org.smartdata.metastore.MetaStoreException; import org.smartdata.metrics.FileAccessEvent; +import org.smartdata.metrics.MetricsFactory; import org.smartdata.model.FileState; import org.smartdata.protocol.ClientServerProto; import org.smartdata.protocol.SmartServerProtocols; @@ -42,13 +45,18 @@ * TODO: Implement statistics for SSM rpc server */ public class SmartRpcServer implements SmartServerProtocols { + private static final String METRICS_PREFIX = "ssm"; + private static final String RPC_METRICS_EXPORTER = "rpcServerMetricsExporter"; + private static final String RPC_METHODS_METRICS_EXPORTER = "rpcServerMethodMetricsExporter"; + protected SmartServer ssm; protected Configuration conf; protected final InetSocketAddress clientRpcAddress; protected int serviceHandlerCount; protected final RPC.Server clientRpcServer; - public SmartRpcServer(SmartServer ssm, Configuration conf) throws IOException { + public SmartRpcServer(SmartServer ssm, + Configuration conf, MetricsFactory metricsFactory) throws IOException { this.ssm = ssm; this.conf = conf; InetSocketAddress rpcAddr = getRpcServerAddress(); @@ -87,13 +95,28 @@ public SmartRpcServer(SmartServer ssm, Configuration conf) throws IOException { if (serviceAuthEnabled) { clientRpcServer.refreshServiceAcl(conf, new SmartPolicyProvider()); } + + initializeMetrics(metricsFactory); + } + + private void initializeMetrics(MetricsFactory metricsFactory) { + MetricsSystem metricsSystem = DefaultMetricsSystem.initialize(METRICS_PREFIX); + + SmartRpcServerMetricsBinder rpcMetricsSink = new SmartRpcServerMetricsBinder(); + rpcMetricsSink.bindTo(metricsFactory.getMeterRegistry()); + metricsSystem.register(RPC_METRICS_EXPORTER, "", rpcMetricsSink); + + SmartRpcServerMethodMetricsBinder detailedRpcMetricsSink = + new SmartRpcServerMethodMetricsBinder(); + detailedRpcMetricsSink.bindTo(metricsFactory.getMeterRegistry()); + metricsSystem.register(RPC_METHODS_METRICS_EXPORTER, "", detailedRpcMetricsSink); } private InetSocketAddress getRpcServerAddress() { String[] strings = conf.get(SmartConfKeys.SMART_SERVER_RPC_ADDRESS_KEY, SmartConfKeys.SMART_SERVER_RPC_ADDRESS_DEFAULT).split(":"); - return new InetSocketAddress(strings[strings.length - 2] - , Integer.parseInt(strings[strings.length - 1])); + return new InetSocketAddress(strings[strings.length - 2], + Integer.parseInt(strings[strings.length - 1])); } /** diff --git a/smart-server/src/main/java/org/smartdata/server/SmartRpcServerMethodMetricsBinder.java b/smart-server/src/main/java/org/smartdata/server/SmartRpcServerMethodMetricsBinder.java new file mode 100644 index 00000000000..ed12fba990e --- /dev/null +++ b/smart-server/src/main/java/org/smartdata/server/SmartRpcServerMethodMetricsBinder.java @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.server; + +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tags; +import io.micrometer.core.instrument.binder.MeterBinder; +import org.apache.commons.configuration2.SubsetConfiguration; +import org.apache.hadoop.metrics2.AbstractMetric; +import org.apache.hadoop.metrics2.MetricsRecord; +import org.apache.hadoop.metrics2.MetricsSink; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Supplier; + +import static org.smartdata.server.SmartServer.SMART_SERVER_BASE_TAGS; + +public class SmartRpcServerMethodMetricsBinder implements MeterBinder, MetricsSink { + private static final String DETAILED_RPC_RECORD_NAME = "rpcdetailed"; + private static final String NUM_OPS_METRIC_POSTFIX = "NumOps"; + private static final String AVG_TIME_METRIC_POSTFIX = "AvgTime"; + + private final Tags baseTags; + private final Map metricsContainer; + + private MeterRegistry registry; + + public SmartRpcServerMethodMetricsBinder() { + this(SMART_SERVER_BASE_TAGS); + } + + public SmartRpcServerMethodMetricsBinder(Tags baseTags) { + this.baseTags = baseTags; + this.metricsContainer = new ConcurrentHashMap<>(); + } + + @Override + public void bindTo(MeterRegistry registry) { + // save reference to the registry for lazy gauge creation + this.registry = registry; + } + + @Override + public void putMetrics(MetricsRecord record) { + if (record.name().equals(DETAILED_RPC_RECORD_NAME)) { + record.metrics().forEach(this::handleMetric); + } + } + + public void handleMetric(AbstractMetric metric) { + metricsContainer.computeIfAbsent(metric.name(), this::createRpcMethodGauge); + metricsContainer.put(metric.name(), metric.value().doubleValue()); + } + + private Double createRpcMethodGauge(String metricName) { + Supplier gaugeProvider = + () -> metricsContainer.getOrDefault(metricName, 0.0); + + if (metricName.endsWith(NUM_OPS_METRIC_POSTFIX)) { + Gauge.builder("rpc.method.calls.total", gaugeProvider) + .tags(Tags.concat(baseTags, "method", + metricName.replace(NUM_OPS_METRIC_POSTFIX, ""))) + .description("Total number of the times the RPC method is called") + .register(registry); + } else { + Gauge.builder("rpc.method.call.time.average.ms", gaugeProvider) + .tags(Tags.concat(baseTags, "method", + metricName.replace(AVG_TIME_METRIC_POSTFIX, ""))) + .description("Average turn around time of the RPC method in milliseconds") + .register(registry); + } + + return 0.0; + } + + @Override + public void flush() { + + } + + @Override + public void init(SubsetConfiguration conf) { + + } +} diff --git a/smart-server/src/main/java/org/smartdata/server/SmartRpcServerMetricsBinder.java b/smart-server/src/main/java/org/smartdata/server/SmartRpcServerMetricsBinder.java new file mode 100644 index 00000000000..dd20053cf0d --- /dev/null +++ b/smart-server/src/main/java/org/smartdata/server/SmartRpcServerMetricsBinder.java @@ -0,0 +1,113 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.smartdata.server; + +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tags; +import io.micrometer.core.instrument.binder.MeterBinder; +import org.apache.commons.configuration2.SubsetConfiguration; +import org.apache.hadoop.metrics2.MetricsRecord; +import org.apache.hadoop.metrics2.MetricsSink; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Supplier; + +import static org.smartdata.server.SmartServer.SMART_SERVER_BASE_TAGS; + +public class SmartRpcServerMetricsBinder implements MeterBinder, MetricsSink { + private static final String RPC_RECORD_NAME = "rpc"; + + private final Tags baseTags; + private final Map metricsContainer; + + public SmartRpcServerMetricsBinder() { + this(SMART_SERVER_BASE_TAGS); + } + + public SmartRpcServerMetricsBinder(Tags baseTags) { + this.baseTags = baseTags; + this.metricsContainer = new ConcurrentHashMap<>(); + } + + @Override + public void bindTo(MeterRegistry registry) { + Gauge.builder("rpc.bytes.received", metricProvider("ReceivedBytes")) + .tags(baseTags) + .description("Total number of received bytes by RPC server") + .register(registry); + + Gauge.builder("rpc.bytes.sent", metricProvider("SentBytes")) + .tags(baseTags) + .description("Total number of sent bytes by RPC server") + .register(registry); + + Gauge.builder("rpc.calls.total", metricProvider("RpcQueueTimeNumOps")) + .tags(baseTags) + .description("Total number of RPC calls") + .register(registry); + + Gauge.builder("rpc.requests.served.total", metricProvider("TotalRequests")) + .tags(baseTags) + .description("Total num of requests served by the RPC server") + .register(registry); + + Gauge.builder("rpc.connections.open", metricProvider("NumOpenConnections")) + .tags(baseTags) + .description("Current number of open RPC connections") + .register(registry); + + Gauge.builder("rpc.call.queue.size", metricProvider("CallQueueLength")) + .tags(baseTags) + .description("Current length of the RPC call queue") + .register(registry); + + Gauge.builder("rpc.call.authentication.success.total", metricProvider("RpcAuthenticationSuccesses")) + .tags(baseTags) + .description("Total number of RPC authentication successes") + .register(registry); + + Gauge.builder("rpc.call.authentication.failure.total", metricProvider("RpcAuthenticationFailures")) + .tags(baseTags) + .description("Total number of RPC authentication failures") + .register(registry); + } + + @Override + public void putMetrics(MetricsRecord record) { + if (record.name().equals(RPC_RECORD_NAME)) { + record.metrics().forEach(metric -> + metricsContainer.put(metric.name(), metric.value().doubleValue())); + } + } + + @Override + public void flush() { + + } + + @Override + public void init(SubsetConfiguration conf) { + + } + + private Supplier metricProvider(String name) { + return () -> metricsContainer.getOrDefault(name, 0.0); + } +} diff --git a/smart-server/src/main/java/org/smartdata/server/SmartServer.java b/smart-server/src/main/java/org/smartdata/server/SmartServer.java index c8f838422b2..a00e17bab6b 100644 --- a/smart-server/src/main/java/org/smartdata/server/SmartServer.java +++ b/smart-server/src/main/java/org/smartdata/server/SmartServer.java @@ -86,8 +86,7 @@ public void initWith(MetaStore metaStore) throws Exception { context = new ServerContext(conf, metaStore, metricsFactory); engine = new SmartEngine(context); - // TODO add RPC metrics - rpcServer = new SmartRpcServer(this, conf); + rpcServer = new SmartRpcServer(this, conf, metricsFactory); restServer = new SmartMasterRestServer(conf, engine); LOG.info("Finish Init Smart Server"); diff --git a/smart-server/src/main/resources/hadoop-metrics2-ssm.properties b/smart-server/src/main/resources/hadoop-metrics2-ssm.properties index 9c6de578cf1..a2e621fca08 100644 --- a/smart-server/src/main/resources/hadoop-metrics2-ssm.properties +++ b/smart-server/src/main/resources/hadoop-metrics2-ssm.properties @@ -2,4 +2,5 @@ # See javadoc of package-info.java for org.apache.hadoop.metrics2 for details *.source.filter.class=org.apache.hadoop.metrics2.filter.GlobFilter -ssm.sink.rpcServerMetricsExporter.record.filter.include=rpc* \ No newline at end of file +ssm.sink.rpcServerMetricsExporter.record.filter.include=rpc +ssm.sink.rpcServerMethodMetricsExporter.record.filter.include=rpcdetailed \ No newline at end of file diff --git a/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/SecurityConfiguration.java b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/SecurityConfiguration.java index 295546722ae..df0495bd3bf 100644 --- a/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/SecurityConfiguration.java +++ b/smart-web-server/smart-web-server-base/src/main/java/org/smartdata/http/config/SecurityConfiguration.java @@ -41,7 +41,7 @@ public class SecurityConfiguration { private static final String SESSION_COOKIE_NAME = "SSM_SESSIONID"; private static final String API_ENDPOINTS_PATTERN = "/api/**"; - private static final String METRICS_ENDPOINT_PATTERN = "/actuator/prometheus"; + private static final String METRICS_ENDPOINT_PATTERN = "/actuator/prometheus/**"; @Bean @ConditionalOnProperty(name = WEB_SECURITY_ENABLED, havingValue = "true") From bef2c9323ed847d49a0b63030942171f2a1e1a68 Mon Sep 17 00:00:00 2001 From: Tigran Manasyan Date: Wed, 20 Nov 2024 13:03:22 +0400 Subject: [PATCH 3/3] [ADH-5361] Actualize Docker demo stand --- .../docker/multihost/conf/smart-default.xml | 40 +++++++++++++++++++ .../tools/docker/multihost/conf/smart-env.sh | 34 ++++++++++++++++ .../docker/multihost/conf/smart-site.xml | 8 ---- .../docker/multihost/docker-compose.yaml | 6 +++ .../docker/multihost/kerberos/kdc-init.sh | 2 - .../multihost/ssm/ssm-server-entrypoint.sh | 8 +--- 6 files changed, 81 insertions(+), 17 deletions(-) create mode 100644 supports/tools/docker/multihost/conf/smart-env.sh diff --git a/supports/tools/docker/multihost/conf/smart-default.xml b/supports/tools/docker/multihost/conf/smart-default.xml index 1493ea8ed04..2e064e26f5d 100644 --- a/supports/tools/docker/multihost/conf/smart-default.xml +++ b/supports/tools/docker/multihost/conf/smart-default.xml @@ -655,4 +655,44 @@ Whether to enable unsuccessful REST server auth attempts. + + + smart.metrics.enabled + true + + Whether to enable metrics collection and export. + + + + + smart.metrics.jmx.enabled + true + + Whether to enable JMX metrics exporter. + + + + + smart.metrics.jmx.domain + metrics + + Default domain for JMX exporter. + + + + + smart.metrics.prometheus.enabled + true + + Whether to enable Prometheus metrics exporter. + + + + + smart.metrics.db.queries.enabled + true + + Whether to enable SQL queries statistics export. + + diff --git a/supports/tools/docker/multihost/conf/smart-env.sh b/supports/tools/docker/multihost/conf/smart-env.sh new file mode 100644 index 00000000000..096ffb9815a --- /dev/null +++ b/supports/tools/docker/multihost/conf/smart-env.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# It's better to have JAVA_HOME configured through this file in the following way +# when starting a distributed SSM cluster. Or in some cases, it may leads to some +# error (cases reported on Ubuntu OS). +#export JAVA_HOME= + + +# Parameters used to start SSM Server JVM +export SSM_SERVER_JAVA_OPT="-Dcom.sun.management.jmxremote.rmi.port=3333 -Djava.rmi.server.hostname=localhost -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.port=3333 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" + +# Parameters used to start SSM Agent JVM +export SSM_AGENT_JAVA_OPT="-Dcom.sun.management.jmxremote.rmi.port=3334 -Djava.rmi.server.hostname=localhost -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.port=3334 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" + +# Common parameters used to start JVM both for SSM Server and Agent +#export SSM_JAVA_OPT= + +# Set Hadoop native lib path for SSM compression use. Generally, the path is $HADOOP_HOME/lib/native +# export LD_LIBRARY_PATH= \ No newline at end of file diff --git a/supports/tools/docker/multihost/conf/smart-site.xml b/supports/tools/docker/multihost/conf/smart-site.xml index b3711db0af9..3a3d091d6fd 100644 --- a/supports/tools/docker/multihost/conf/smart-site.xml +++ b/supports/tools/docker/multihost/conf/smart-site.xml @@ -25,14 +25,6 @@ smart.server.rpc.address ssm-server.demo:7042 - - smart.agent.address - hadoop-datanode.demo:7048 - - - smart.agent.master.address - hadoop-datanode.demo:7051 - smart.rule.executors 5 diff --git a/supports/tools/docker/multihost/docker-compose.yaml b/supports/tools/docker/multihost/docker-compose.yaml index 83396fff013..3e08b072f25 100644 --- a/supports/tools/docker/multihost/docker-compose.yaml +++ b/supports/tools/docker/multihost/docker-compose.yaml @@ -43,6 +43,10 @@ services: - "9864:9864" - "7048:7048" - "7051:7051" + # SSM Agent web server port + - "18081:8081" + # JMX port + - "3334:3334" env_file: - hadoop.env healthcheck: @@ -67,6 +71,8 @@ services: ports: - "7042:7042" - "8081:8081" + # JMX port + - "3333:3333" healthcheck: test: curl http://ssm-server.demo:8081 || exit 1 interval: 30s diff --git a/supports/tools/docker/multihost/kerberos/kdc-init.sh b/supports/tools/docker/multihost/kerberos/kdc-init.sh index 54fe099988a..f8b9fbcd858 100644 --- a/supports/tools/docker/multihost/kerberos/kdc-init.sh +++ b/supports/tools/docker/multihost/kerberos/kdc-init.sh @@ -29,7 +29,6 @@ kadmin.local -q "add_principal -randkey HTTP/ssm-server.demo@DEMO" kadmin.local -q "add_principal -randkey HTTP/hadoop-datanode.demo@DEMO" kadmin.local -q "add_principal -randkey ssm/ssm-server.demo@DEMO" kadmin.local -q "add_principal -randkey agent/hadoop-datanode.demo@DEMO" -kadmin.local -q "add_principal -randkey agent/ssm-server.demo@DEMO" kadmin.local -q "add_principal -randkey yarn/hadoop-namenode.demo@DEMO" kadmin.local -q "add_principal -pw krb_pass1 krb_user1@DEMO" kadmin.local -q "add_principal -pw krb_pass2 krb_user2@DEMO" @@ -46,7 +45,6 @@ kadmin.local -q "xst -kt /tmp/secrets/http.keytab HTTP/ssm-server.demo@DEMO" && kadmin.local -q "xst -kt /tmp/secrets/http.keytab HTTP/hadoop-datanode.demo@DEMO" && chown appuser:appuser /tmp/secrets/http.keytab kadmin.local -q "xst -kt /tmp/secrets/ssm.keytab ssm/ssm-server.demo@DEMO" && chown appuser:appuser /tmp/secrets/ssm.keytab kadmin.local -q "xst -kt /tmp/secrets/agent.keytab agent/hadoop-datanode.demo@DEMO" && chown appuser:appuser /tmp/secrets/agent.keytab -kadmin.local -q "xst -kt /tmp/secrets/agent.keytab agent/ssm-server.demo@DEMO" && chown appuser:appuser /tmp/secrets/agent.keytab kadmin.local -q "xst -kt /tmp/secrets/yarn.keytab yarn/hadoop-namenode.demo@DEMO" && chown appuser:appuser /tmp/secrets/yarn.keytab echo "" diff --git a/supports/tools/docker/multihost/ssm/ssm-server-entrypoint.sh b/supports/tools/docker/multihost/ssm/ssm-server-entrypoint.sh index 49f46ad72d9..50447a28037 100644 --- a/supports/tools/docker/multihost/ssm/ssm-server-entrypoint.sh +++ b/supports/tools/docker/multihost/ssm/ssm-server-entrypoint.sh @@ -14,17 +14,11 @@ echo "export SMART_CONF_DIR=${SSM_HOME}/conf/" >> /root/.bashrc cd $SSM_HOME || exit echo "---------------------------" -echo "Starting SSM server locally" +echo "Starting SSM server and agents" echo "---------------------------" source bin/start-ssm.sh --config ${SSM_HOME}/conf/ & wait_for_it $(hostname -f):8081 - -echo "-------------------" -echo "Starting SSM agents" -echo "-------------------" - -source bin/start-agent.sh & wait_for_it hadoop-datanode.demo:7048 tail -f /var/log/ssm/* \ No newline at end of file