Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Limit amount of extension versions #888

Closed
wants to merge 1 commit into from
+359 −85
Closed

Limit amount of extension versions #888

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions server/src/main/java/org/eclipse/openvsx/admin/AdminService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -371,4 +371,15 @@ private void validateYearAndMonth(int year, int month) {
throw new ErrorResultException("Combination of year and month lies in the future", HttpStatus.BAD_REQUEST);
}
}

@Transactional
public UserData createSystemUser(String userName) {
var user = repositories.findUserByLoginName(null, userName);
if(user == null) {
user = new UserData();
user.setLoginName(userName);
entityManager.persist(user);
}
return user;
}
}
6 changes: 2 additions & 4 deletions server/src/main/java/org/eclipse/openvsx/migration/FixTargetPlatformsJobRequestHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,6 @@ public class FixTargetPlatformsJobRequestHandler implements JobRequestHandler<Mi
@Autowired
MigrationService migrations;

@Autowired
FixTargetPlatformsService service;

@Override
@Job(name = "Fix target platform for published extension version", retries = 3)
public void run(MigrationJobRequest jobRequest) throws Exception {
Expand Down Expand Up @@ -71,13 +68,14 @@ public void run(MigrationJobRequest jobRequest) throws Exception {
}

private void deleteExtension(ExtensionVersion extVersion) {
var user = admins.createSystemUser("FixTargetPlatformMigration");
var extension = extVersion.getExtension();
admins.deleteExtension(
extension.getNamespace().getName(),
extension.getName(),
extVersion.getTargetPlatform(),
extVersion.getVersion(),
service.getUser()
user
);
}
}
40 changes: 0 additions & 40 deletions server/src/main/java/org/eclipse/openvsx/migration/FixTargetPlatformsService.java
View file
Open in desktop

This file was deleted.

9 changes: 1 addition & 8 deletions server/src/main/java/org/eclipse/openvsx/mirror/DataMirrorService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,15 +139,8 @@ public List<ExtensionVersion> getExtensionTargetVersions(String namespaceName, S
return extension.getVersions().stream().filter(v -> targetPlatform.equals(v.getTargetPlatform())).collect(Collectors.toList());
}

@Transactional
public UserData createMirrorUser() {
var user = repositories.findUserByLoginName(null, userName);
if(user == null) {
user = new UserData();
user.setLoginName(userName);
entityManager.persist(user);
}
return user;
return admin.createSystemUser(userName);
}

@Transactional
Expand Down
4 changes: 2 additions & 2 deletions server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
package org.eclipse.openvsx.publish;

import com.google.common.base.Joiner;
import jakarta.persistence.EntityManager;
import jakarta.transaction.Transactional;
import org.apache.commons.lang3.StringUtils;
import org.eclipse.openvsx.ExtensionProcessor;
import org.eclipse.openvsx.ExtensionService;
Expand All @@ -29,8 +31,6 @@
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Component;

import jakarta.persistence.EntityManager;
import jakarta.transaction.Transactional;
import java.io.IOException;
import java.time.LocalDateTime;
import java.util.function.Consumer;
Expand Down
20 changes: 20 additions & 0 deletions server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,12 @@
import org.eclipse.openvsx.entities.ExtensionVersion;
import org.eclipse.openvsx.entities.FileResource;
import org.eclipse.openvsx.repositories.RepositoryService;
import org.eclipse.openvsx.storage.LimitStoredVersionsJobRequest;
import org.eclipse.openvsx.storage.StorageUtilService;
import org.eclipse.openvsx.storage.StoredVersionsLimiter;
import org.eclipse.openvsx.util.ErrorResultException;
import org.eclipse.openvsx.util.TempFile;
import org.jobrunr.scheduling.JobRequestScheduler;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.annotation.CacheEvict;
import org.springframework.retry.annotation.Retryable;
Expand All @@ -41,6 +44,12 @@ public class PublishExtensionVersionService {
@Autowired
StorageUtilService storageUtil;

@Autowired
StoredVersionsLimiter storedVersionsLimiter;

@Autowired
JobRequestScheduler scheduler;

@Transactional
public void deleteFileResources(ExtensionVersion extVersion) {
repositories.findFiles(extVersion).forEach(entityManager::remove);
Expand Down Expand Up @@ -92,6 +101,17 @@ public void activateExtension(ExtensionVersion extVersion, ExtensionService exte
extVersion.setActive(true);
extVersion = entityManager.merge(extVersion);
extensions.updateExtension(extVersion.getExtension());
scheduleLimitStoredVersionsJob(extVersion);
}

private void scheduleLimitStoredVersionsJob(ExtensionVersion extVersion) {
if(!storedVersionsLimiter.isEnabled()) {
return;
}

var extension = extVersion.getExtension();
var namespace = extension.getNamespace();
scheduler.enqueue(new LimitStoredVersionsJobRequest(namespace.getName(), extension.getName()));
}

@Transactional(Transactional.TxType.REQUIRES_NEW)
Expand Down
152 changes: 122 additions & 30 deletions server/src/main/java/org/eclipse/openvsx/repositories/ExtensionVersionJooqRepository.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@

import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import static org.eclipse.openvsx.jooq.Tables.*;

Expand Down Expand Up @@ -543,42 +544,46 @@ private List<String> toList(String raw, ListOfStringConverter converter) {
return converter.convertToEntityAttribute(raw);
}

private SelectQuery<Record> findGroupedByVersionQuery() {
var query = dsl.selectQuery();
query.addSelect(
EXTENSION_VERSION.SEMVER_MAJOR,
EXTENSION_VERSION.SEMVER_MINOR,
EXTENSION_VERSION.SEMVER_PATCH,
EXTENSION_VERSION.SEMVER_IS_PRE_RELEASE,
EXTENSION_VERSION.VERSION
);
query.addFrom(EXTENSION_VERSION);
query.addGroupBy(
EXTENSION_VERSION.SEMVER_MAJOR,
EXTENSION_VERSION.SEMVER_MINOR,
EXTENSION_VERSION.SEMVER_PATCH,
EXTENSION_VERSION.SEMVER_IS_PRE_RELEASE,
EXTENSION_VERSION.VERSION
);
query.addOrderBy(
EXTENSION_VERSION.SEMVER_MAJOR.desc(),
EXTENSION_VERSION.SEMVER_MINOR.desc(),
EXTENSION_VERSION.SEMVER_PATCH.desc(),
EXTENSION_VERSION.SEMVER_IS_PRE_RELEASE.asc(),
EXTENSION_VERSION.VERSION.asc()
);
return query;
}

public List<VersionTargetPlatformsJson> findTargetPlatformsGroupedByVersion(Extension extension) {
var query = findGroupedByVersionQuery();
var targetPlatforms = DSL.arrayAgg(EXTENSION_VERSION.TARGET_PLATFORM)
.orderBy(
EXTENSION_VERSION.UNIVERSAL_TARGET_PLATFORM.desc(),
EXTENSION_VERSION.TARGET_PLATFORM.asc()
);

return dsl.select(
EXTENSION_VERSION.SEMVER_MAJOR,
EXTENSION_VERSION.SEMVER_MINOR,
EXTENSION_VERSION.SEMVER_PATCH,
EXTENSION_VERSION.SEMVER_IS_PRE_RELEASE,
EXTENSION_VERSION.VERSION,
targetPlatforms
)
.from(EXTENSION_VERSION)
.where(EXTENSION_VERSION.EXTENSION_ID.eq(extension.getId()))
.groupBy(
EXTENSION_VERSION.SEMVER_MAJOR,
EXTENSION_VERSION.SEMVER_MINOR,
EXTENSION_VERSION.SEMVER_PATCH,
EXTENSION_VERSION.SEMVER_IS_PRE_RELEASE,
EXTENSION_VERSION.VERSION
)
.orderBy(
EXTENSION_VERSION.SEMVER_MAJOR.desc(),
EXTENSION_VERSION.SEMVER_MINOR.desc(),
EXTENSION_VERSION.SEMVER_PATCH.desc(),
EXTENSION_VERSION.SEMVER_IS_PRE_RELEASE.asc(),
EXTENSION_VERSION.VERSION.asc()
)
.fetch()
.map(record -> new VersionTargetPlatformsJson(
record.get(EXTENSION_VERSION.VERSION),
record.get(targetPlatforms)
));
query.addSelect(targetPlatforms);
query.addConditions(EXTENSION_VERSION.EXTENSION_ID.eq(extension.getId()));
return query.fetch(record -> new VersionTargetPlatformsJson(
record.get(EXTENSION_VERSION.VERSION),
record.get(targetPlatforms)
));
}

@Observed
Expand Down Expand Up @@ -792,4 +797,91 @@ public List<String> findDistinctTargetPlatforms(Extension extension) {
.and(EXTENSION_VERSION.ACTIVE.eq(true))
.fetch(EXTENSION_VERSION.TARGET_PLATFORM);
}

public List<ExtensionVersion> findExcess(String namespaceName, String extensionName, int limit) {
var TARGET_PLATFORMS = DSL.arrayAgg(EXTENSION_VERSION.TARGET_PLATFORM);
var query = findGroupedByVersionQuery();
query.addSelect(TARGET_PLATFORMS);
query.addJoin(EXTENSION, EXTENSION.ID.eq(EXTENSION_VERSION.EXTENSION_ID));
query.addJoin(NAMESPACE, NAMESPACE.ID.eq(EXTENSION.NAMESPACE_ID));
query.addConditions(
NAMESPACE.NAME.eq(namespaceName),
EXTENSION.NAME.eq(extensionName)
);
query.addOffset(limit);
return query.fetch(record -> {
var list = new ArrayList<ExtensionVersion>();
var version = record.get(EXTENSION_VERSION.VERSION);
var targetPlatforms = record.get(TARGET_PLATFORMS);
for(var targetPlatform : targetPlatforms) {
var namespace = new Namespace();
namespace.setName(namespaceName);

var extension = new Extension();
extension.setName(extensionName);
extension.setNamespace(namespace);

var extVersion = new ExtensionVersion();
extVersion.setVersion(version);
extVersion.setTargetPlatform(targetPlatform);
extVersion.setExtension(extension);
list.add(extVersion);
}
return list;
})
.stream()
.flatMap(List::stream)
.collect(Collectors.toList());
}

public List<ExtensionVersion> findAllExcess(int limit) {
var namespaceNameAlias = NAMESPACE.NAME.as("namespaceName");
var extensionNameAlias = EXTENSION.NAME.as("extensionName");
var table = dsl.select(namespaceNameAlias, extensionNameAlias, EXTENSION.ID)
.from(EXTENSION)
.join(NAMESPACE).on(NAMESPACE.ID.eq(EXTENSION.NAMESPACE_ID))
.asTable();

var namespaceNameField = table.field(namespaceNameAlias);
var extensionNameField = table.field(extensionNameAlias);

var TARGET_PLATFORMS = DSL.arrayAgg(EXTENSION_VERSION.TARGET_PLATFORM);
var lateralQuery = findGroupedByVersionQuery();
lateralQuery.addSelect(TARGET_PLATFORMS);
lateralQuery.addConditions(EXTENSION_VERSION.EXTENSION_ID.eq(table.field(EXTENSION.ID)));
lateralQuery.addOffset(limit);
var lateralTable = lateralQuery.asTable();
var versionField = lateralTable.field(EXTENSION_VERSION.VERSION);
var targetPlatformsField = lateralTable.field(TARGET_PLATFORMS);

var query = dsl.selectQuery();
query.addSelect(namespaceNameField, extensionNameField, versionField, targetPlatformsField);
query.addFrom(table, DSL.lateral(lateralTable));

return query.fetch(record -> {
var list = new ArrayList<ExtensionVersion>();
var namespaceName = record.get(namespaceNameField);
var extensionName = record.get(extensionNameField);
var version = record.get(versionField);
var targetPlatforms = record.get(targetPlatformsField);
for(var targetPlatform : targetPlatforms) {
var namespace = new Namespace();
namespace.setName(namespaceName);

var extension = new Extension();
extension.setName(extensionName);
extension.setNamespace(namespace);

var extVersion = new ExtensionVersion();
extVersion.setVersion(version);
extVersion.setTargetPlatform(targetPlatform);
extVersion.setExtension(extension);
list.add(extVersion);
}
return list;
})
.stream()
.flatMap(List::stream)
.collect(Collectors.toList());
}
}
8 changes: 8 additions & 0 deletions server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -555,4 +555,12 @@ public ExtensionVersion findLatestVersion(Extension extension, String targetPlat
public List<String> findExtensionTargetPlatforms(Extension extension) {
return extensionVersionJooqRepo.findDistinctTargetPlatforms(extension);
}

public List<ExtensionVersion> findExcessExtensionVersions(String namespace, String extension, int limit) {
return extensionVersionJooqRepo.findExcess(namespace, extension, limit);
}

public List<ExtensionVersion> findAllExcessExtensionVersions(int limit) {
return extensionVersionJooqRepo.findAllExcess(limit);
}
}
Loading
Loading