diff --git a/driver/src/main/java/eu/cloudnetservice/driver/service/ServiceRemoteInclusion.java b/driver/src/main/java/eu/cloudnetservice/driver/service/ServiceRemoteInclusion.java index 90efa1154e..58322e21fd 100644 --- a/driver/src/main/java/eu/cloudnetservice/driver/service/ServiceRemoteInclusion.java +++ b/driver/src/main/java/eu/cloudnetservice/driver/service/ServiceRemoteInclusion.java @@ -22,8 +22,9 @@ import eu.cloudnetservice.driver.document.property.DocProperty; import io.leangen.geantyref.TypeFactory; import java.util.Map; -import lombok.EqualsAndHashCode; +import java.util.Objects; import lombok.NonNull; +import org.jetbrains.annotations.Nullable; /** * An inclusion which can be added to a service and will download a file from the specified url to the given @@ -31,9 +32,11 @@ * * @since 4.0 */ -@EqualsAndHashCode public final class ServiceRemoteInclusion implements DefaultedDocPropertyHolder, Cloneable { + public static final String NO_CACHE_STRATEGY = "none"; + public static final String KEEP_UNTIL_RESTART_STRATEGY = "until-node-restart"; + /** * A property which can be added to a service inclusion to set the http headers to send when making the download http * request. All key-value pairs of the given document will be set as headers in the request. @@ -44,19 +47,28 @@ public final class ServiceRemoteInclusion implements DefaultedDocPropertyHolder, private final String url; private final String destination; + private final String cacheStrategy; private final Document properties; /** * Constructs a new service remote inclusion instance. * - * @param url the url to download the associated file from. - * @param destination the destination inside the service directory to copy the downloaded file to. - * @param properties the properties of the remote inclusion, these can for example contain the http headers to send. + * @param url the url to download the associated file from. + * @param destination the destination inside the service directory to copy the downloaded file to. + * @param cacheStrategy the cache strategy to use when downloading files from the remote. + * @param properties the properties of the remote inclusion, these can for example contain the http headers to + * send. * @throws NullPointerException if one of the given parameters is null. */ - private ServiceRemoteInclusion(@NonNull String url, @NonNull String destination, @NonNull Document properties) { + private ServiceRemoteInclusion( + @NonNull String url, + @NonNull String destination, + @NonNull String cacheStrategy, + @NonNull Document properties + ) { this.url = url; this.destination = destination; + this.cacheStrategy = cacheStrategy; this.properties = properties; } @@ -84,6 +96,7 @@ private ServiceRemoteInclusion(@NonNull String url, @NonNull String destination, return builder() .url(inclusion.url()) .destination(inclusion.destination()) + .cacheStrategy(inclusion.cacheStrategy()) .properties(inclusion.propertyHolder()); } @@ -106,6 +119,15 @@ private ServiceRemoteInclusion(@NonNull String url, @NonNull String destination, return this.destination; } + /** + * The caching strategy that is used when downloading the inclusion. + * + * @return the caching strategy. + */ + public @NonNull String cacheStrategy() { + return this.cacheStrategy; + } + /** * {@inheritDoc} */ @@ -134,6 +156,25 @@ private ServiceRemoteInclusion(@NonNull String url, @NonNull String destination, } } + /** + * {@inheritDoc} + */ + @Override + public boolean equals(@Nullable Object other) { + if (this == other) { + return true; + } + if (!(other instanceof ServiceRemoteInclusion that)) { + return false; + } + return Objects.equals(this.url, that.url()) && Objects.equals(this.destination, that.destination()); + } + + @Override + public int hashCode() { + return Objects.hash(this.url, this.destination); + } + /** * A builder for a service remote inclusion. * @@ -143,6 +184,7 @@ public static class Builder { protected String url; protected String destination; + protected String cacheStrategy = ServiceRemoteInclusion.NO_CACHE_STRATEGY; protected Document properties = Document.emptyDocument(); /** @@ -172,6 +214,21 @@ public static class Builder { return this; } + /** + * Sets the cache strategy that is used when downloading the inclusion from the remote. + *

+ * To disable caching use the {@link ServiceRemoteInclusion#NO_CACHE_STRATEGY}, which is also the default of this + * builder. + * + * @param cacheStrategy the caching strategy to use. + * @return the same instance as used to call the method, for chaining. + * @throws NullPointerException if the given cache strategy is null. + */ + public @NonNull Builder cacheStrategy(@NonNull String cacheStrategy) { + this.cacheStrategy = cacheStrategy; + return this; + } + /** * Sets the properties of the service remote inclusion. The properties can for example be used to set the http * headers which should get send when making a request to the given download url. @@ -189,13 +246,14 @@ public static class Builder { * Builds a service remote inclusion instance based on this builder. * * @return the service remote inclusion. - * @throws NullPointerException if no url or destination was given. + * @throws NullPointerException if no url, destination or cache strategy is given. */ public @NonNull ServiceRemoteInclusion build() { Preconditions.checkNotNull(this.url, "no url given"); Preconditions.checkNotNull(this.destination, "no destination given"); + Preconditions.checkNotNull(this.cacheStrategy, "no cacheStrategy given"); - return new ServiceRemoteInclusion(this.url, this.destination, this.properties); + return new ServiceRemoteInclusion(this.url, this.destination, this.cacheStrategy, this.properties); } } } diff --git a/node/src/main/java/eu/cloudnetservice/node/command/sub/GroupsCommand.java b/node/src/main/java/eu/cloudnetservice/node/command/sub/GroupsCommand.java index c39bce4dd2..d8643a9e16 100644 --- a/node/src/main/java/eu/cloudnetservice/node/command/sub/GroupsCommand.java +++ b/node/src/main/java/eu/cloudnetservice/node/command/sub/GroupsCommand.java @@ -76,6 +76,22 @@ public GroupsCommand(@NonNull GroupConfigurationProvider groupProvider) { return this.groupProvider.groupConfigurations().stream().map(Named::name).toList(); } + @Parser(name = "inclusionCacheStrategy", suggestions = "inclusionCacheStrategy") + public @NonNull String inclusionCacheStrategyParser(@NonNull CommandContext $, @NonNull Queue input) { + var strategy = input.remove(); + if (strategy.equals(ServiceRemoteInclusion.NO_CACHE_STRATEGY) || + strategy.equals(ServiceRemoteInclusion.KEEP_UNTIL_RESTART_STRATEGY)) { + return strategy; + } + + throw new ArgumentNotAvailableException(I18n.trans("command-tasks-inclusion-cache-strategy-not-found", strategy)); + } + + @Suggestions("inclusionCacheStrategy") + public @NonNull List inclusionCacheStrategySuggester(@NonNull CommandContext $, @NonNull String input) { + return List.of(ServiceRemoteInclusion.NO_CACHE_STRATEGY, ServiceRemoteInclusion.KEEP_UNTIL_RESTART_STRATEGY); + } + @CommandMethod("groups delete ") public void deleteGroup(@NonNull CommandSource source, @NonNull @Argument("name") GroupConfiguration configuration) { this.groupProvider.removeGroupConfiguration(configuration); @@ -186,14 +202,22 @@ public void addTemplate( group.name())); } - @CommandMethod("groups group add inclusion ") + @CommandMethod("groups group add inclusion [cacheStrategy]") public void addInclusion( @NonNull CommandSource source, @NonNull @Argument("name") GroupConfiguration group, @NonNull @Argument("url") String url, - @NonNull @Argument("path") String path + @NonNull @Argument("path") String path, + @NonNull @Argument( + value = "cacheStrategy", + parserName = "inclusionCacheStrategy", + defaultValue = ServiceRemoteInclusion.NO_CACHE_STRATEGY) String cacheStrategy ) { - var inclusion = ServiceRemoteInclusion.builder().url(url).destination(path).build(); + var inclusion = ServiceRemoteInclusion.builder() + .url(url) + .destination(path) + .cacheStrategy(cacheStrategy) + .build(); this.updateGroup(group, builder -> builder.modifyInclusions(inclusions -> inclusions.add(inclusion))); source.sendMessage(I18n.trans("command-groups-add-collection-property", "inclusion", @@ -344,6 +368,17 @@ public void clearProcessParameters( group.name())); } + @CommandMethod("groups group clear inclusions") + public void clearInclusions( + @NonNull CommandSource source, + @NonNull @Argument("name") GroupConfiguration group + ) { + this.updateGroup(group, builder -> builder.modifyInclusions(Collection::clear)); + source.sendMessage(I18n.trans("command-groups-clear-property", + "inclusions", + group.name())); + } + private void updateGroup(@NonNull GroupConfiguration group, Consumer modifier) { modifier .andThen(builder -> this.groupProvider.addGroupConfiguration(builder.build())) diff --git a/node/src/main/java/eu/cloudnetservice/node/command/sub/TasksCommand.java b/node/src/main/java/eu/cloudnetservice/node/command/sub/TasksCommand.java index 83ac25e837..07fa87ed1b 100644 --- a/node/src/main/java/eu/cloudnetservice/node/command/sub/TasksCommand.java +++ b/node/src/main/java/eu/cloudnetservice/node/command/sub/TasksCommand.java @@ -133,7 +133,7 @@ public static void applyServiceConfigurationDisplay( messages.add("Includes:"); for (var inclusion : configurationBase.inclusions()) { - messages.add("- " + inclusion.url() + " => " + inclusion.destination()); + messages.add("- " + inclusion.url() + ':' + inclusion.cacheStrategy() + " => " + inclusion.destination()); } messages.add(" "); @@ -634,18 +634,22 @@ public void addTemplate( template); } - @CommandMethod("tasks task add inclusion ") + @CommandMethod("tasks task add inclusion [cacheStrategy]") public void addInclusion( @NonNull CommandSource source, @NonNull @Argument("name") Collection tasks, @NonNull @Argument("url") String url, - @NonNull @Argument("path") String path + @NonNull @Argument("path") String path, + @NonNull @Argument( + value = "cacheStrategy", + parserName = "inclusionCacheStrategy", + defaultValue = ServiceRemoteInclusion.NO_CACHE_STRATEGY) String cacheStrategy ) { - var inclusion = ServiceRemoteInclusion.builder().url(url).destination(path).build(); + var inclusion = ServiceRemoteInclusion.builder().url(url).destination(path).cacheStrategy(cacheStrategy).build(); this.applyChange( source, tasks, - (builder, $) -> builder.modifyInclusions(col -> col.add(inclusion)), + (builder, _) -> builder.modifyInclusions(col -> col.add(inclusion)), "command-tasks-add-collection-property", "inclusion", inclusion); @@ -829,6 +833,20 @@ public void clearProcessParameter( null); } + @CommandMethod("tasks task clear inclusions") + public void clearInclusions( + @NonNull CommandSource source, + @NonNull @Argument("name") Collection tasks + ) { + this.applyChange( + source, + tasks, + (builder, $) -> builder.modifyInclusions(Collection::clear), + "command-tasks-clear-property", + "inclusions", + null); + } + @CommandMethod("tasks task unset javaCommand") public void unsetJavaCommand( @NonNull CommandSource source, diff --git a/node/src/main/java/eu/cloudnetservice/node/event/service/CloudServicePreLoadInclusionEvent.java b/node/src/main/java/eu/cloudnetservice/node/event/service/CloudServicePreLoadInclusionEvent.java index c943a19725..8bff5b8328 100644 --- a/node/src/main/java/eu/cloudnetservice/node/event/service/CloudServicePreLoadInclusionEvent.java +++ b/node/src/main/java/eu/cloudnetservice/node/event/service/CloudServicePreLoadInclusionEvent.java @@ -19,33 +19,28 @@ import eu.cloudnetservice.driver.event.Cancelable; import eu.cloudnetservice.driver.service.ServiceRemoteInclusion; import eu.cloudnetservice.node.service.CloudService; -import kong.unirest.core.GetRequest; import lombok.NonNull; public final class CloudServicePreLoadInclusionEvent extends CloudServiceEvent implements Cancelable { - private final GetRequest request; - private final ServiceRemoteInclusion serviceRemoteInclusion; - - private volatile boolean cancelled; + private boolean cancelled; + private ServiceRemoteInclusion serviceRemoteInclusion; public CloudServicePreLoadInclusionEvent( @NonNull CloudService cloudService, - @NonNull ServiceRemoteInclusion serviceRemoteInclusion, - @NonNull GetRequest request + @NonNull ServiceRemoteInclusion serviceRemoteInclusion ) { super(cloudService); this.serviceRemoteInclusion = serviceRemoteInclusion; - this.request = request; } public @NonNull ServiceRemoteInclusion inclusion() { return this.serviceRemoteInclusion; } - public @NonNull GetRequest request() { - return this.request; + public void inclusion(@NonNull ServiceRemoteInclusion inclusion) { + this.serviceRemoteInclusion = inclusion; } public boolean cancelled() { diff --git a/node/src/main/java/eu/cloudnetservice/node/provider/NodeGroupConfigurationProvider.java b/node/src/main/java/eu/cloudnetservice/node/provider/NodeGroupConfigurationProvider.java index 9779cd22ec..08bc446934 100644 --- a/node/src/main/java/eu/cloudnetservice/node/provider/NodeGroupConfigurationProvider.java +++ b/node/src/main/java/eu/cloudnetservice/node/provider/NodeGroupConfigurationProvider.java @@ -30,6 +30,7 @@ import eu.cloudnetservice.driver.network.rpc.handler.RPCHandlerRegistry; import eu.cloudnetservice.driver.provider.GroupConfigurationProvider; import eu.cloudnetservice.driver.service.GroupConfiguration; +import eu.cloudnetservice.driver.service.ServiceRemoteInclusion; import eu.cloudnetservice.node.cluster.sync.DataSyncHandler; import eu.cloudnetservice.node.cluster.sync.DataSyncRegistry; import eu.cloudnetservice.node.event.group.LocalGroupConfigurationAddEvent; @@ -45,6 +46,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import lombok.NonNull; @@ -60,6 +62,7 @@ public class NodeGroupConfigurationProvider implements GroupConfigurationProvide private static final Path GROUP_DIRECTORY_PATH = Path.of( System.getProperty("cloudnet.config.groups.directory.path", "local/groups")); + private static final Type LIST_DOCUMENT_TYPE = TypeFactory.parameterizedClass(List.class, Document.class); private static final Type TYPE = TypeFactory.parameterizedClass(Collection.class, GroupConfiguration.class); private final EventManager eventManager; @@ -228,6 +231,18 @@ protected void loadGroupConfigurations() { document.append("environmentVariables", new HashMap<>()); } + // inclusions now feature a mandatory cache strategy field. + // Convert old inclusions that do not have a non-null value already set. + List remoteInclusions = document.readObject("includes", LIST_DOCUMENT_TYPE); + var migratedInclusions = remoteInclusions.stream().map(remoteInclusion -> { + if (remoteInclusion.containsNonNull("cacheStrategy")) { + return remoteInclusion; + } + + return remoteInclusion.mutableCopy().append("cacheStrategy", ServiceRemoteInclusion.NO_CACHE_STRATEGY); + }).toList(); + document.append("includes", migratedInclusions); + // load the group var group = document.toInstanceOf(GroupConfiguration.class); diff --git a/node/src/main/java/eu/cloudnetservice/node/provider/NodeServiceTaskProvider.java b/node/src/main/java/eu/cloudnetservice/node/provider/NodeServiceTaskProvider.java index d297886a75..616151a5a2 100644 --- a/node/src/main/java/eu/cloudnetservice/node/provider/NodeServiceTaskProvider.java +++ b/node/src/main/java/eu/cloudnetservice/node/provider/NodeServiceTaskProvider.java @@ -31,6 +31,7 @@ import eu.cloudnetservice.driver.network.rpc.factory.RPCFactory; import eu.cloudnetservice.driver.network.rpc.handler.RPCHandlerRegistry; import eu.cloudnetservice.driver.provider.ServiceTaskProvider; +import eu.cloudnetservice.driver.service.ServiceRemoteInclusion; import eu.cloudnetservice.driver.service.ServiceTask; import eu.cloudnetservice.node.cluster.sync.DataSyncHandler; import eu.cloudnetservice.node.cluster.sync.DataSyncRegistry; @@ -40,14 +41,17 @@ import eu.cloudnetservice.node.setup.DefaultInstallation; import eu.cloudnetservice.node.setup.DefaultTaskSetup; import eu.cloudnetservice.node.util.JavaVersionResolver; +import io.leangen.geantyref.TypeFactory; import jakarta.inject.Inject; import jakarta.inject.Singleton; +import java.lang.reflect.Type; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import lombok.NonNull; @@ -61,6 +65,7 @@ public class NodeServiceTaskProvider implements ServiceTaskProvider { private static final Path TASKS_DIRECTORY = Path.of( System.getProperty("cloudnet.config.tasks.directory.path", "local/tasks")); + private static final Type LIST_DOCUMENT_TYPE = TypeFactory.parameterizedClass(List.class, Document.class); private static final Logger LOGGER = LoggerFactory.getLogger(NodeServiceTaskProvider.class); @@ -204,6 +209,18 @@ protected void loadServiceTasks() { document.append("processConfiguration", processConfiguration); } + // inclusions now feature a mandatory cache strategy field. + // Convert old inclusions that do not have a non-null value already set. + List remoteInclusions = document.readObject("includes", LIST_DOCUMENT_TYPE); + var migratedInclusions = remoteInclusions.stream().map(remoteInclusion -> { + if (remoteInclusion.containsNonNull("cacheStrategy")) { + return remoteInclusion; + } + + return remoteInclusion.mutableCopy().append("cacheStrategy", ServiceRemoteInclusion.NO_CACHE_STRATEGY); + }).toList(); + document.append("includes", migratedInclusions); + // load the service task var task = document.toInstanceOf(ServiceTask.class); // check if the file name is still up-to-date diff --git a/node/src/main/java/eu/cloudnetservice/node/service/defaults/AbstractService.java b/node/src/main/java/eu/cloudnetservice/node/service/defaults/AbstractService.java index 02b3a5e8aa..81e2ae921f 100644 --- a/node/src/main/java/eu/cloudnetservice/node/service/defaults/AbstractService.java +++ b/node/src/main/java/eu/cloudnetservice/node/service/defaults/AbstractService.java @@ -17,6 +17,7 @@ package eu.cloudnetservice.node.service.defaults; import com.google.common.base.Preconditions; +import com.google.common.hash.Hashing; import com.google.common.net.InetAddresses; import eu.cloudnetservice.common.io.FileUtil; import eu.cloudnetservice.common.language.I18n; @@ -63,7 +64,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; -import java.util.Base64; import java.util.Collection; import java.util.List; import java.util.Map; @@ -357,43 +357,41 @@ public void includeWaitingServiceTemplates(boolean force) { public void includeWaitingServiceInclusions() { ServiceRemoteInclusion inclusion; while ((inclusion = this.waitingRemoteInclusions.poll()) != null) { - // prepare the connection from which we load the inclusion - var req = Unirest.get(inclusion.url()); - // put the given http headers - var headers = inclusion.readPropertyOrDefault(ServiceRemoteInclusion.HEADERS, Map.of()); - for (var entry : headers.entrySet()) { - req.header(entry.getKey(), entry.getValue()); - } - // check if we should load the inclusion - if (!this.eventManager.callEvent(new CloudServicePreLoadInclusionEvent(this, inclusion, req)).cancelled()) { - // get a target path based on the download url - var encodedUrl = Base64.getEncoder().encodeToString(inclusion.url().getBytes(StandardCharsets.UTF_8)); - var destination = INCLUSION_TEMP_DIR.resolve(encodedUrl.replace('/', '_')); - - // download the file from the given url to the temp path if it does not exist - if (Files.notExists(destination)) { - try { - // copy the file to the temp path, ensure that the parent directory exists - FileUtil.createDirectory(INCLUSION_TEMP_DIR); - req.asFile(destination.toString(), StandardCopyOption.REPLACE_EXISTING); - } catch (UnirestException exception) { - LOGGER.error( - "Unable to download inclusion from {} to {}", - inclusion.url(), - destination, - exception.getCause()); - continue; - } - } - + var preLoadEvent = this.eventManager.callEvent(new CloudServicePreLoadInclusionEvent(this, inclusion)); + if (!preLoadEvent.cancelled()) { + // the event might have changed the inclusion, use the updated one + inclusion = preLoadEvent.inclusion(); // resolve the desired output path var target = this.serviceDirectory.resolve(inclusion.destination()); FileUtil.ensureChild(this.serviceDirectory, target); - // copy the file to the desired output path - FileUtil.copy(destination, target); - // we've installed the inclusion successfully - this.installedInclusions.add(inclusion); + + try { + if (inclusion.cacheStrategy().equals(ServiceRemoteInclusion.KEEP_UNTIL_RESTART_STRATEGY)) { + // get a target path based on the download url + var encodedUrl = Hashing.murmur3_128().hashString(inclusion.url(), StandardCharsets.UTF_8).toString(); + var destination = INCLUSION_TEMP_DIR.resolve(encodedUrl); + // download the file to the temp path if it does not exist + if (Files.notExists(destination)) { + this.downloadInclusionFile(inclusion, destination); + } + + // copy the file from the temp path to the desired output path + FileUtil.copy(destination, target); + } else { + // download the file directly to the target path if caching is disabled + this.downloadInclusionFile(inclusion, target); + } + + // we've installed the inclusion successfully + this.installedInclusions.add(inclusion); + } catch (UnirestException exception) { + LOGGER.warn( + "Unable to download inclusion from {} to {}", + inclusion.url(), + target, + exception.getCause()); + } } } } @@ -763,6 +761,19 @@ protected void prepareService() { .append("trustCertificatePath", relativeFilePath.toString()); } + protected void downloadInclusionFile(@NonNull ServiceRemoteInclusion inclusion, @NonNull Path destination) { + // prepare the connection from which we load the inclusion + var request = Unirest.get(inclusion.url()); + // put the given http headers + var headers = inclusion.readPropertyOrDefault(ServiceRemoteInclusion.HEADERS, Map.of()); + for (var entry : headers.entrySet()) { + request.header(entry.getKey(), entry.getValue()); + } + + FileUtil.createDirectory(destination.getParent()); + request.asFile(destination.toString(), StandardCopyOption.REPLACE_EXISTING); + } + protected @NonNull Object[] serviceReplacement() { return new Object[]{ this.serviceId().uniqueId(), diff --git a/node/src/main/resources/lang/en_US.properties b/node/src/main/resources/lang/en_US.properties index 684c540807..caf5829190 100644 --- a/node/src/main/resources/lang/en_US.properties +++ b/node/src/main/resources/lang/en_US.properties @@ -266,6 +266,7 @@ command-tasks-create-task=The empty task was successfully created. Configure it command-tasks-delete-task=The task {0$name$} was successfully deleted command-tasks-node-not-found=That node doesn't exist! command-tasks-runtime-not-found=The runtime {0$runtime$} does not exist! +command-tasks-inclusion-cache-strategy-not-found=The inclusion cache strategy {0$strategy$} does not exist! command-tasks-reload-success=The ServiceTasks have been reloaded! command-tasks-add-collection-property=The {0$property$} {2$value$} was successfully added to the task {1$task$} command-tasks-set-property-success=The {0$property$} of the task {1$task$} was changed to {2$value$}