diff --git a/server/src/main/java/org/eclipse/openvsx/ExtensionService.java b/server/src/main/java/org/eclipse/openvsx/ExtensionService.java index 1caa5d052..913bda83b 100644 --- a/server/src/main/java/org/eclipse/openvsx/ExtensionService.java +++ b/server/src/main/java/org/eclipse/openvsx/ExtensionService.java @@ -22,12 +22,14 @@ import org.apache.commons.lang3.StringUtils; import org.eclipse.openvsx.cache.CacheService; import org.eclipse.openvsx.entities.*; -import org.eclipse.openvsx.publish.PublishExtensionVersionHandler; +import org.eclipse.openvsx.publish.PublishExtensionVersionJobRequest; +import org.eclipse.openvsx.publish.PublishExtensionVersionService; import org.eclipse.openvsx.repositories.RepositoryService; import org.eclipse.openvsx.search.SearchUtilService; import org.eclipse.openvsx.util.ErrorResultException; import org.eclipse.openvsx.util.TempFile; import org.eclipse.openvsx.util.TimeUtil; +import org.jobrunr.scheduling.JobRequestScheduler; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpStatus; @@ -48,7 +50,10 @@ public class ExtensionService { CacheService cache; @Autowired - PublishExtensionVersionHandler publishHandler; + PublishExtensionVersionService publishService; + + @Autowired + JobRequestScheduler scheduler; @Value("${ovsx.publishing.require-license:false}") boolean requireLicense; @@ -56,20 +61,29 @@ public class ExtensionService { @Transactional public ExtensionVersion mirrorVersion(TempFile extensionFile, String signatureName, PersonalAccessToken token, String binaryName, String timestamp) { var download = doPublish(extensionFile, binaryName, token, TimeUtil.fromUTCString(timestamp), false); - publishHandler.mirror(download, extensionFile, signatureName); + publishService.mirror(download, extensionFile, signatureName); return download.getExtension(); } public ExtensionVersion publishVersion(InputStream content, PersonalAccessToken token) { - var extensionFile = createExtensionFile(content); - var download = doPublish(extensionFile, null, token, TimeUtil.getCurrentUTC(), true); - publishHandler.publishAsync(download, extensionFile, this); - return download.getExtension(); + try(var extensionFile = createExtensionFile(content)) { + var download = doPublish(extensionFile, null, token, TimeUtil.getCurrentUTC(), true); + download.setContent(Files.readAllBytes(extensionFile.getPath())); + download.setStorageType(FileResource.STORAGE_DB); + publishService.persistDownload(download); + + var jobRequest = new PublishExtensionVersionJobRequest(download.getId()); + scheduler.enqueue(jobRequest); + + return download.getExtension(); + } catch (IOException e) { + throw new ErrorResultException("failed to read extension VSIX package", e); + } } private FileResource doPublish(TempFile extensionFile, String binaryName, PersonalAccessToken token, LocalDateTime timestamp, boolean checkDependencies) { try (var processor = new ExtensionProcessor(extensionFile)) { - var extVersion = publishHandler.createExtensionVersion(processor, token, timestamp, checkDependencies); + var extVersion = publishService.createExtensionVersion(processor, token, timestamp, checkDependencies); if (requireLicense) { // Check the extension's license var license = processor.getLicense(extVersion); diff --git a/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java b/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java index bcfefb34c..876120e8a 100644 --- a/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java +++ b/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java @@ -640,7 +640,7 @@ public ExtensionJson publish(InputStream content, String tokenValue) throws Erro var extVersion = extensions.publishVersion(content, token); var json = toExtensionVersionJson(extVersion, null, true, true); - json.success = "It can take a couple minutes before the extension version is available"; + json.success = "It can take a couple minutes before the extension version becomes active"; var sameVersions = repositories.findVersions(extVersion.getVersion(), extVersion.getExtension()); if(sameVersions.stream().anyMatch(ev -> ev.isPreRelease() != extVersion.isPreRelease())) { diff --git a/server/src/main/java/org/eclipse/openvsx/adapter/VSCodeIdService.java b/server/src/main/java/org/eclipse/openvsx/adapter/VSCodeIdService.java index e69e541da..67be05946 100644 --- a/server/src/main/java/org/eclipse/openvsx/adapter/VSCodeIdService.java +++ b/server/src/main/java/org/eclipse/openvsx/adapter/VSCodeIdService.java @@ -10,6 +10,8 @@ package org.eclipse.openvsx.adapter; import com.google.common.collect.Lists; +import jakarta.persistence.EntityManager; +import jakarta.transaction.Transactional; import org.apache.commons.lang3.StringUtils; import org.eclipse.openvsx.UrlConfigService; import org.eclipse.openvsx.entities.Extension; @@ -43,6 +45,14 @@ public class VSCodeIdService { @Autowired UrlConfigService urlConfigService; + @Autowired + EntityManager entityManager; + + @Transactional(Transactional.TxType.REQUIRES_NEW) + public void updateExtensionPublicId(Extension extension) { + entityManager.merge(extension); + } + public boolean setPublicIds(Extension extension) { var updateExistingPublicIds = false; var upstream = getUpstreamExtension(extension); diff --git a/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionHandler.java b/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionHandler.java deleted file mode 100644 index 6ebcd6989..000000000 --- a/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionHandler.java +++ /dev/null @@ -1,268 +0,0 @@ -/** ****************************************************************************** - * Copyright (c) 2022 Precies. Software and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Public License v. 2.0 which is available at - * http://www.eclipse.org/legal/epl-2.0. - * - * SPDX-License-Identifier: EPL-2.0 - * ****************************************************************************** */ -package org.eclipse.openvsx.publish; - -import com.google.common.base.Joiner; -import org.eclipse.openvsx.ExtensionProcessor; -import org.eclipse.openvsx.ExtensionService; -import org.eclipse.openvsx.ExtensionValidator; -import org.eclipse.openvsx.UserService; -import org.eclipse.openvsx.adapter.VSCodeIdService; -import org.eclipse.openvsx.entities.*; -import org.eclipse.openvsx.repositories.RepositoryService; -import org.eclipse.openvsx.util.ErrorResultException; -import org.eclipse.openvsx.util.NamingUtil; -import org.eclipse.openvsx.util.TempFile; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.retry.annotation.Retryable; -import org.springframework.scheduling.annotation.Async; -import org.springframework.stereotype.Component; - -import jakarta.persistence.EntityManager; -import jakarta.transaction.Transactional; -import java.io.IOException; -import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.function.Consumer; -import java.util.stream.Collectors; - -@Component -public class PublishExtensionVersionHandler { - - protected final Logger logger = LoggerFactory.getLogger(PublishExtensionVersionHandler.class); - - @Autowired - PublishExtensionVersionService service; - - @Autowired - ExtensionVersionIntegrityService integrityService; - - @Autowired - EntityManager entityManager; - - @Autowired - RepositoryService repositories; - - @Autowired - VSCodeIdService vsCodeIdService; - - @Autowired - UserService users; - - @Autowired - ExtensionValidator validator; - - @Transactional - public ExtensionVersion createExtensionVersion(ExtensionProcessor processor, PersonalAccessToken token, LocalDateTime timestamp, boolean checkDependencies) { - // Extract extension metadata from its manifest - var extVersion = createExtensionVersion(processor, token.getUser(), token, timestamp); - var dependencies = processor.getExtensionDependencies(); - var bundledExtensions = processor.getBundledExtensions(); - if (checkDependencies) { - dependencies = dependencies.stream() - .map(this::checkDependency) - .collect(Collectors.toList()); - bundledExtensions = bundledExtensions.stream() - .map(this::checkBundledExtension) - .collect(Collectors.toList()); - } - - extVersion.setDependencies(dependencies); - extVersion.setBundledExtensions(bundledExtensions); - if(integrityService.isEnabled()) { - extVersion.setSignatureKeyPair(repositories.findActiveKeyPair()); - } - - return extVersion; - } - - private ExtensionVersion createExtensionVersion(ExtensionProcessor processor, UserData user, PersonalAccessToken token, LocalDateTime timestamp) { - var namespaceName = processor.getNamespace(); - var namespace = repositories.findNamespace(namespaceName); - if (namespace == null) { - throw new ErrorResultException("Unknown publisher: " + namespaceName - + "\nUse the 'create-namespace' command to create a namespace corresponding to your publisher name."); - } - if (!users.hasPublishPermission(user, namespace)) { - throw new ErrorResultException("Insufficient access rights for publisher: " + namespace.getName()); - } - - var extensionName = processor.getExtensionName(); - var nameIssue = validator.validateExtensionName(extensionName); - if (nameIssue.isPresent()) { - throw new ErrorResultException(nameIssue.get().toString()); - } - - var versionIssue = validator.validateExtensionVersion(processor.getVersion()); - if (versionIssue.isPresent()) { - throw new ErrorResultException(versionIssue.get().toString()); - } - - var extVersion = processor.getMetadata(); - if (extVersion.getDisplayName() != null && extVersion.getDisplayName().trim().isEmpty()) { - extVersion.setDisplayName(null); - } - extVersion.setTimestamp(timestamp); - extVersion.setPublishedWith(token); - extVersion.setActive(false); - - var extension = repositories.findExtension(extensionName, namespace); - if (extension == null) { - extension = new Extension(); - extension.setActive(false); - extension.setName(extensionName); - extension.setNamespace(namespace); - extension.setPublishedDate(extVersion.getTimestamp()); - - var updateExistingPublicIds = vsCodeIdService.setPublicIds(extension); - if(updateExistingPublicIds) { - updateExistingPublicIds(extension).forEach(service::updateExtensionPublicId); - } - - entityManager.persist(extension); - } else { - var existingVersion = repositories.findVersion(extVersion.getVersion(), extVersion.getTargetPlatform(), extension); - if (existingVersion != null) { - var extVersionId = NamingUtil.toLogFormat(namespaceName, extensionName, extVersion.getTargetPlatform(), extVersion.getVersion()); - var message = "Extension " + extVersionId + " is already published"; - message += existingVersion.isActive() ? "." : ", but currently isn't active and therefore not visible."; - throw new ErrorResultException(message); - } - } - - extension.setLastUpdatedDate(extVersion.getTimestamp()); - extension.getVersions().add(extVersion); - extVersion.setExtension(extension); - - var metadataIssues = validator.validateMetadata(extVersion); - if (!metadataIssues.isEmpty()) { - if (metadataIssues.size() == 1) { - throw new ErrorResultException(metadataIssues.get(0).toString()); - } - throw new ErrorResultException("Multiple issues were found in the extension metadata:\n" - + Joiner.on("\n").join(metadataIssues)); - } - - entityManager.persist(extVersion); - return extVersion; - } - - private String checkDependency(String dependency) { - var split = dependency.split("\\."); - if (split.length != 2 || split[0].isEmpty() || split[1].isEmpty()) { - throw new ErrorResultException("Invalid 'extensionDependencies' format. Expected: '${namespace}.${name}'"); - } - var extensionCount = repositories.countExtensions(split[1], split[0]); - if (extensionCount == 0) { - throw new ErrorResultException("Cannot resolve dependency: " + dependency); - } - - return dependency; - } - - private String checkBundledExtension(String bundledExtension) { - var split = bundledExtension.split("\\."); - if (split.length != 2 || split[0].isEmpty() || split[1].isEmpty()) { - throw new ErrorResultException("Invalid 'extensionPack' format. Expected: '${namespace}.${name}'"); - } - - return bundledExtension; - } - - private List updateExistingPublicIds(Extension extension) { - var updated = true; - var updatedExtensions = new ArrayList(); - var newExtension = extension; - while(updated) { - updated = false; - var oldExtension = repositories.findExtensionByPublicId(newExtension.getPublicId()); - if (oldExtension != null && !oldExtension.equals(newExtension)) { - entityManager.detach(oldExtension); - updated = vsCodeIdService.setPublicIds(oldExtension); - } - if(updated) { - updatedExtensions.add(oldExtension); - newExtension = oldExtension; - } - } - - Collections.reverse(updatedExtensions); - return updatedExtensions; - } - - @Async - @Retryable - public void publishAsync(FileResource download, TempFile extensionFile, ExtensionService extensionService) { - var extVersion = download.getExtension(); - - // Delete file resources in case publishAsync is retried - service.deleteFileResources(extVersion); - download.setId(0L); - - service.storeDownload(download, extensionFile); - service.persistResource(download); - try(var processor = new ExtensionProcessor(extensionFile)) { - Consumer consumer = resource -> { - service.storeResource(resource); - service.persistResource(resource); - }; - - if(integrityService.isEnabled()) { - var keyPair = extVersion.getSignatureKeyPair(); - if(keyPair != null) { - var signature = integrityService.generateSignature(download, extensionFile, keyPair); - consumer.accept(signature); - } else { - // Can happen when GenerateKeyPairJobRequestHandler hasn't run yet and there is no active SignatureKeyPair. - // This extension version should be assigned a SignatureKeyPair and a signature FileResource should be created - // by the ExtensionVersionSignatureJobRequestHandler migration. - logger.warn("Integrity service is enabled, but {} did not have an active key pair", NamingUtil.toLogFormat(extVersion)); - } - } - - processor.processEachResource(extVersion, consumer); - processor.getFileResources(extVersion).forEach(consumer); - consumer.accept(processor.generateSha256Checksum(extVersion)); - } - - // Update whether extension is active, the search index and evict cache - service.activateExtension(extVersion, extensionService); - try { - extensionFile.close(); - } catch (IOException e) { - logger.error("failed to delete temp file", e); - } - } - - public void mirror(FileResource download, TempFile extensionFile, String signatureName) { - var extVersion = download.getExtension(); - service.mirrorResource(download); - if(signatureName != null) { - service.mirrorResource(getSignatureResource(signatureName, extVersion)); - } - try(var processor = new ExtensionProcessor(extensionFile)) { - processor.getFileResources(extVersion).forEach(resource -> service.mirrorResource(resource)); - service.mirrorResource(processor.generateSha256Checksum(extVersion)); - // don't store file resources, they can be generated on the fly to avoid traversing entire zip file - } - } - - private FileResource getSignatureResource(String signatureName, ExtensionVersion extVersion) { - var resource = new FileResource(); - resource.setExtension(extVersion); - resource.setName(signatureName); - resource.setType(FileResource.DOWNLOAD_SIG); - return resource; - } -} diff --git a/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionJobRequest.java b/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionJobRequest.java new file mode 100644 index 000000000..6e14cd369 --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionJobRequest.java @@ -0,0 +1,37 @@ +/** ****************************************************************************** + * Copyright (c) 2023 Precies. Software Ltd and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + * ****************************************************************************** */ +package org.eclipse.openvsx.publish; + +import org.jobrunr.jobs.lambdas.JobRequest; +import org.jobrunr.jobs.lambdas.JobRequestHandler; + +public class PublishExtensionVersionJobRequest implements JobRequest { + + private long downloadId; + + public PublishExtensionVersionJobRequest() {} + + public PublishExtensionVersionJobRequest(long downloadId) { + this.downloadId = downloadId; + } + + @Override + public Class getJobRequestHandler() { + return PublishExtensionVersionJobRequestHandler.class; + } + + public long getDownloadId() { + return downloadId; + } + + public void setDownloadId(long downloadId) { + this.downloadId = downloadId; + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionJobRequestHandler.java b/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionJobRequestHandler.java new file mode 100644 index 000000000..1f5948cca --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionJobRequestHandler.java @@ -0,0 +1,84 @@ +/** ****************************************************************************** + * Copyright (c) 2023 Precies. Software Ltd and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + * ****************************************************************************** */ +package org.eclipse.openvsx.publish; + +import org.eclipse.openvsx.ExtensionProcessor; +import org.eclipse.openvsx.ExtensionService; +import org.eclipse.openvsx.entities.FileResource; +import org.eclipse.openvsx.migration.MigrationService; +import org.eclipse.openvsx.util.NamingUtil; +import org.eclipse.openvsx.util.TempFile; +import org.jobrunr.jobs.annotations.Job; +import org.jobrunr.jobs.lambdas.JobRequestHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.nio.file.Files; +import java.util.function.Consumer; + +@Component +public class PublishExtensionVersionJobRequestHandler implements JobRequestHandler { + + protected final Logger logger = LoggerFactory.getLogger(PublishExtensionVersionJobRequestHandler.class); + + @Autowired + ExtensionService extensions; + + @Autowired + ExtensionVersionIntegrityService integrityService; + + @Autowired + PublishExtensionVersionService service; + + @Autowired + MigrationService migrations; + + @Override + @Job(name = "Process extension version file resources", retries = 10) + public void run(PublishExtensionVersionJobRequest jobRequest) throws Exception { + var download = service.getDownload(jobRequest); + var extVersion = download.getExtension(); + + // Delete file resources in case this job is retried + service.deleteFileResources(extVersion); + try (var extensionFile = new TempFile("extension_", ".vsix")) { + Files.write(extensionFile.getPath(), migrations.getContent(download)); + try (var processor = new ExtensionProcessor(extensionFile)) { + service.storeDownload(download); + Consumer consumer = resource -> { + service.storeResource(resource); + service.persistResource(resource); + }; + + if (integrityService.isEnabled()) { + var keyPair = extVersion.getSignatureKeyPair(); + if (keyPair != null) { + var signature = integrityService.generateSignature(download, extensionFile, keyPair); + consumer.accept(signature); + } else { + // Can happen when GenerateKeyPairJobRequestHandler hasn't run yet and there is no active SignatureKeyPair. + // This extension version should be assigned a SignatureKeyPair and a signature FileResource should be created + // by the ExtensionVersionSignatureJobRequestHandler migration. + logger.warn("Integrity service is enabled, but {} did not have an active key pair", NamingUtil.toLogFormat(extVersion)); + } + } + + processor.processEachResource(extVersion, consumer); + processor.getFileResources(extVersion).forEach(consumer); + consumer.accept(processor.generateSha256Checksum(extVersion)); + } + } + + // Update whether extension is active, the search index and evict cache + service.activateExtension(extVersion, extensions); + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionService.java b/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionService.java index 464cf8674..01aba4320 100644 --- a/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionService.java +++ b/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionService.java @@ -9,22 +9,29 @@ * ****************************************************************************** */ package org.eclipse.openvsx.publish; +import com.google.common.base.Joiner; +import jakarta.persistence.EntityManager; +import jakarta.transaction.Transactional; +import org.eclipse.openvsx.ExtensionProcessor; import org.eclipse.openvsx.ExtensionService; -import org.eclipse.openvsx.entities.Extension; -import org.eclipse.openvsx.entities.ExtensionVersion; -import org.eclipse.openvsx.entities.FileResource; +import org.eclipse.openvsx.ExtensionValidator; +import org.eclipse.openvsx.UserService; +import org.eclipse.openvsx.adapter.VSCodeIdService; +import org.eclipse.openvsx.entities.*; import org.eclipse.openvsx.repositories.RepositoryService; import org.eclipse.openvsx.storage.StorageUtilService; import org.eclipse.openvsx.util.ErrorResultException; +import org.eclipse.openvsx.util.NamingUtil; import org.eclipse.openvsx.util.TempFile; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.retry.annotation.Retryable; import org.springframework.stereotype.Component; -import jakarta.persistence.EntityManager; -import jakarta.transaction.Transactional; -import java.io.IOException; -import java.nio.file.Files; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; @Component public class PublishExtensionVersionService { @@ -38,22 +45,168 @@ public class PublishExtensionVersionService { @Autowired StorageUtilService storageUtil; + @Autowired + ExtensionVersionIntegrityService integrityService; + + @Autowired + UserService users; + + @Autowired + ExtensionValidator validator; + + @Autowired + VSCodeIdService vsCodeIdService; + @Transactional - public void deleteFileResources(ExtensionVersion extVersion) { - repositories.findFiles(extVersion).forEach(entityManager::remove); + public ExtensionVersion createExtensionVersion(ExtensionProcessor processor, PersonalAccessToken token, LocalDateTime timestamp, boolean checkDependencies) { + // Extract extension metadata from its manifest + var extVersion = createExtensionVersion(processor, token.getUser(), token, timestamp); + var dependencies = processor.getExtensionDependencies(); + var bundledExtensions = processor.getBundledExtensions(); + if (checkDependencies) { + dependencies = dependencies.stream() + .map(this::checkDependency) + .collect(Collectors.toList()); + bundledExtensions = bundledExtensions.stream() + .map(this::checkBundledExtension) + .collect(Collectors.toList()); + } + + extVersion.setDependencies(dependencies); + extVersion.setBundledExtensions(bundledExtensions); + if(integrityService.isEnabled()) { + extVersion.setSignatureKeyPair(repositories.findActiveKeyPair()); + } + + return extVersion; } - public void storeDownload(FileResource download, TempFile extensionFile) { - if (storageUtil.shouldStoreExternally(download)) { - storageUtil.uploadFile(download, extensionFile); + private ExtensionVersion createExtensionVersion(ExtensionProcessor processor, UserData user, PersonalAccessToken token, LocalDateTime timestamp) { + var namespaceName = processor.getNamespace(); + var namespace = repositories.findNamespace(namespaceName); + if (namespace == null) { + throw new ErrorResultException("Unknown publisher: " + namespaceName + + "\nUse the 'create-namespace' command to create a namespace corresponding to your publisher name."); + } + if (!users.hasPublishPermission(user, namespace)) { + throw new ErrorResultException("Insufficient access rights for publisher: " + namespace.getName()); + } + + var extensionName = processor.getExtensionName(); + var nameIssue = validator.validateExtensionName(extensionName); + if (nameIssue.isPresent()) { + throw new ErrorResultException(nameIssue.get().toString()); + } + + var versionIssue = validator.validateExtensionVersion(processor.getVersion()); + if (versionIssue.isPresent()) { + throw new ErrorResultException(versionIssue.get().toString()); + } + + var extVersion = processor.getMetadata(); + if (extVersion.getDisplayName() != null && extVersion.getDisplayName().trim().isEmpty()) { + extVersion.setDisplayName(null); + } + extVersion.setTimestamp(timestamp); + extVersion.setPublishedWith(token); + extVersion.setActive(false); + + var extension = repositories.findExtension(extensionName, namespace); + if (extension == null) { + extension = new Extension(); + extension.setActive(false); + extension.setName(extensionName); + extension.setNamespace(namespace); + extension.setPublishedDate(extVersion.getTimestamp()); + + var updateExistingPublicIds = vsCodeIdService.setPublicIds(extension); + if(updateExistingPublicIds) { + updateExistingPublicIds(extension).forEach(vsCodeIdService::updateExtensionPublicId); + } + + entityManager.persist(extension); } else { - try { - download.setContent(Files.readAllBytes(extensionFile.getPath())); - } catch (IOException e) { - throw new ErrorResultException("Failed to read extension file", e); + var existingVersion = repositories.findVersion(extVersion.getVersion(), extVersion.getTargetPlatform(), extension); + if (existingVersion != null) { + var extVersionId = NamingUtil.toLogFormat(namespaceName, extensionName, extVersion.getTargetPlatform(), extVersion.getVersion()); + var message = "Extension " + extVersionId + " is already published"; + message += existingVersion.isActive() ? "." : ", but currently is deactivated and therefore not visible."; + throw new ErrorResultException(message); + } + } + + extension.setLastUpdatedDate(extVersion.getTimestamp()); + extension.getVersions().add(extVersion); + extVersion.setExtension(extension); + + var metadataIssues = validator.validateMetadata(extVersion); + if (!metadataIssues.isEmpty()) { + if (metadataIssues.size() == 1) { + throw new ErrorResultException(metadataIssues.get(0).toString()); } + throw new ErrorResultException("Multiple issues were found in the extension metadata:\n" + + Joiner.on("\n").join(metadataIssues)); + } + + entityManager.persist(extVersion); + return extVersion; + } + + private String checkDependency(String dependency) { + var split = dependency.split("\\."); + if (split.length != 2 || split[0].isEmpty() || split[1].isEmpty()) { + throw new ErrorResultException("Invalid 'extensionDependencies' format. Expected: '${namespace}.${name}'"); + } + var extensionCount = repositories.countExtensions(split[1], split[0]); + if (extensionCount == 0) { + throw new ErrorResultException("Cannot resolve dependency: " + dependency); + } + + return dependency; + } + + private String checkBundledExtension(String bundledExtension) { + var split = bundledExtension.split("\\."); + if (split.length != 2 || split[0].isEmpty() || split[1].isEmpty()) { + throw new ErrorResultException("Invalid 'extensionPack' format. Expected: '${namespace}.${name}'"); + } + + return bundledExtension; + } + + private List updateExistingPublicIds(Extension extension) { + var updated = true; + var updatedExtensions = new ArrayList(); + var newExtension = extension; + while(updated) { + updated = false; + var oldExtension = repositories.findExtensionByPublicId(newExtension.getPublicId()); + if (oldExtension != null && !oldExtension.equals(newExtension)) { + entityManager.detach(oldExtension); + updated = vsCodeIdService.setPublicIds(oldExtension); + } + if(updated) { + updatedExtensions.add(oldExtension); + newExtension = oldExtension; + } + } + + Collections.reverse(updatedExtensions); + return updatedExtensions; + } + + @Transactional + public void deleteFileResources(ExtensionVersion extVersion) { + repositories.findFiles(extVersion) + .filter(resource -> !resource.getType().equals(FileResource.DOWNLOAD)) + .forEach(entityManager::remove); + } - download.setStorageType(FileResource.STORAGE_DB); + @Transactional + public void storeDownload(FileResource download) { + if (storageUtil.shouldStoreExternally(download)) { + storageUtil.uploadFile(download); + download.setContent(null); } } @@ -70,19 +223,45 @@ public void storeResource(FileResource resource) { } @Transactional - public void mirrorResource(FileResource resource) { + public void mirror(FileResource download, TempFile extensionFile, String signatureName) { + var extVersion = download.getExtension(); + mirrorResource(download); + if(signatureName != null) { + mirrorResource(getSignatureResource(signatureName, extVersion)); + } + try(var processor = new ExtensionProcessor(extensionFile)) { + processor.getFileResources(extVersion).forEach(resource -> mirrorResource(resource)); + mirrorResource(processor.generateSha256Checksum(extVersion)); + // don't store file resources, they can be generated on the fly to avoid traversing entire zip file + } + } + + private void mirrorResource(FileResource resource) { resource.setStorageType(storageUtil.getActiveStorageType()); // Don't store the binary content in the DB - it's now stored externally resource.setContent(null); entityManager.persist(resource); } + private FileResource getSignatureResource(String signatureName, ExtensionVersion extVersion) { + var resource = new FileResource(); + resource.setExtension(extVersion); + resource.setName(signatureName); + resource.setType(FileResource.DOWNLOAD_SIG); + return resource; + } + @Transactional public void persistResource(FileResource resource) { - resource.setExtension(entityManager.merge(resource.getExtension())); entityManager.persist(resource); } + @Transactional + public void persistDownload(FileResource download) { + persistResource(download); + entityManager.flush(); + } + @Transactional public void activateExtension(ExtensionVersion extVersion, ExtensionService extensions) { extVersion.setActive(true); @@ -90,8 +269,8 @@ public void activateExtension(ExtensionVersion extVersion, ExtensionService exte extensions.updateExtension(extVersion.getExtension()); } - @Transactional(Transactional.TxType.REQUIRES_NEW) - public void updateExtensionPublicId(Extension extension) { - entityManager.merge(extension); + @Transactional + public FileResource getDownload(PublishExtensionVersionJobRequest jobRequest) { + return entityManager.find(FileResource.class, jobRequest.getDownloadId()); } } diff --git a/server/src/test/java/org/eclipse/openvsx/RegistryAPITest.java b/server/src/test/java/org/eclipse/openvsx/RegistryAPITest.java index dcab83182..531a6f1a5 100644 --- a/server/src/test/java/org/eclipse/openvsx/RegistryAPITest.java +++ b/server/src/test/java/org/eclipse/openvsx/RegistryAPITest.java @@ -21,7 +21,6 @@ import org.eclipse.openvsx.entities.*; import org.eclipse.openvsx.json.*; import org.eclipse.openvsx.publish.ExtensionVersionIntegrityService; -import org.eclipse.openvsx.publish.PublishExtensionVersionHandler; import org.eclipse.openvsx.publish.PublishExtensionVersionService; import org.eclipse.openvsx.repositories.RepositoryService; import org.eclipse.openvsx.search.ExtensionSearch; @@ -36,6 +35,7 @@ import org.eclipse.openvsx.storage.StorageUtilService; import org.eclipse.openvsx.util.TargetPlatform; import org.eclipse.openvsx.util.VersionService; +import org.jobrunr.scheduling.JobRequestScheduler; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -84,7 +84,7 @@ @MockBean({ ClientRegistrationRepository.class, UpstreamRegistryService.class, GoogleCloudStorageService.class, AzureBlobStorageService.class, VSCodeIdService.class, AzureDownloadCountService.class, CacheService.class, - EclipseService.class, PublishExtensionVersionService.class, SimpleMeterRegistry.class + EclipseService.class, SimpleMeterRegistry.class, JobRequestScheduler.class }) public class RegistryAPITest { @@ -2314,8 +2314,8 @@ LatestExtensionVersionCacheKeyGenerator latestExtensionVersionCacheKeyGenerator( } @Bean - PublishExtensionVersionHandler publishExtensionVersionHandler() { - return new PublishExtensionVersionHandler(); + PublishExtensionVersionService publishExtensionVersionService() { + return new PublishExtensionVersionService(); } } } \ No newline at end of file diff --git a/server/src/test/java/org/eclipse/openvsx/admin/AdminAPITest.java b/server/src/test/java/org/eclipse/openvsx/admin/AdminAPITest.java index d0a0c8a40..c865cf2d1 100644 --- a/server/src/test/java/org/eclipse/openvsx/admin/AdminAPITest.java +++ b/server/src/test/java/org/eclipse/openvsx/admin/AdminAPITest.java @@ -20,7 +20,7 @@ import org.eclipse.openvsx.entities.*; import org.eclipse.openvsx.json.*; import org.eclipse.openvsx.publish.ExtensionVersionIntegrityService; -import org.eclipse.openvsx.publish.PublishExtensionVersionHandler; +import org.eclipse.openvsx.publish.PublishExtensionVersionService; import org.eclipse.openvsx.repositories.RepositoryService; import org.eclipse.openvsx.search.SearchUtilService; import org.eclipse.openvsx.security.OAuth2UserServices; @@ -71,7 +71,7 @@ @MockBean({ ClientRegistrationRepository.class, UpstreamRegistryService.class, GoogleCloudStorageService.class, AzureBlobStorageService.class, VSCodeIdService.class, AzureDownloadCountService.class, - CacheService.class, PublishExtensionVersionHandler.class, SearchUtilService.class, + CacheService.class, PublishExtensionVersionService.class, SearchUtilService.class, EclipseService.class, SimpleMeterRegistry.class }) public class AdminAPITest { diff --git a/server/src/test/java/org/eclipse/openvsx/eclipse/EclipseServiceTest.java b/server/src/test/java/org/eclipse/openvsx/eclipse/EclipseServiceTest.java index a8769c7b1..6ec8e8d0e 100644 --- a/server/src/test/java/org/eclipse/openvsx/eclipse/EclipseServiceTest.java +++ b/server/src/test/java/org/eclipse/openvsx/eclipse/EclipseServiceTest.java @@ -28,7 +28,7 @@ import org.eclipse.openvsx.cache.CacheService; import org.eclipse.openvsx.cache.LatestExtensionVersionCacheKeyGenerator; import org.eclipse.openvsx.entities.*; -import org.eclipse.openvsx.publish.PublishExtensionVersionHandler; +import org.eclipse.openvsx.publish.PublishExtensionVersionService; import org.eclipse.openvsx.repositories.RepositoryService; import org.eclipse.openvsx.search.SearchUtilService; import org.eclipse.openvsx.security.TokenService; @@ -39,6 +39,7 @@ import org.eclipse.openvsx.util.ErrorResultException; import org.eclipse.openvsx.util.TargetPlatform; import org.eclipse.openvsx.util.VersionService; +import org.jobrunr.scheduling.JobRequestScheduler; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -65,9 +66,8 @@ @ExtendWith(SpringExtension.class) @MockBean({ EntityManager.class, SearchUtilService.class, GoogleCloudStorageService.class, AzureBlobStorageService.class, - VSCodeIdService.class, AzureDownloadCountService.class, CacheService.class, - UserService.class, PublishExtensionVersionHandler.class, - SimpleMeterRegistry.class + VSCodeIdService.class, AzureDownloadCountService.class, CacheService.class, UserService.class, + PublishExtensionVersionService.class, SimpleMeterRegistry.class, JobRequestScheduler.class }) public class EclipseServiceTest {