From c74a717898bfd9d17ca8de5d9d08201296738a43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E6=B2=BB=E5=9B=BD?= Date: Mon, 29 Jan 2024 20:00:21 +0800 Subject: [PATCH 1/4] 123 --- bigtop-manager-agent/pom.xml | 1 + .../common/message/type/pojo/ClusterInfo.java | 3 +- bigtop-manager-server/pom.xml | 1 + .../server/holder/SpringContextHolder.java | 5 + .../job/factory/AbstractJobFactory.java | 4 + .../cluster/ClusterCreateJobFactory.java | 3 - .../AbstractComponentJobFactory.java | 4 - .../job/factory/host/HostAddJobFactory.java | 5 +- .../service/ServiceInstallJobFactory.java | 5 + .../job/helper/HostCacheStageHelper.java | 2 - .../job/helper/HostCheckStageHelper.java | 2 - .../job/listener/CommandEventListener.java | 5 +- .../job/stage/AbstractStageLifecycle.java | 24 ++ .../stage/ComponentInstallStageLifecycle.java | 22 ++ .../job/stage/HostCacheStageLifecycle.java | 253 ++++++++++++++++++ .../job/stage/HostCheckStageLifecycle.java | 97 +++++++ .../server/job/stage/StageLifecycle.java | 40 +++ .../job/stage/StageLifecycleContext.java | 26 ++ .../server/job/stage/StageLifecycles.java | 45 ++++ .../manager/server/job/stage/StageType.java | 21 ++ .../server/model/mapper/JobMapper.java | 20 +- .../bigtop/manager/server/orm/entity/Job.java | 5 + bigtop-manager-ui/src/api/cluster/index.ts | 11 +- 23 files changed, 555 insertions(+), 49 deletions(-) create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/AbstractStageLifecycle.java create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/ComponentInstallStageLifecycle.java create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/HostCacheStageLifecycle.java create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/HostCheckStageLifecycle.java create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/StageLifecycle.java create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/StageLifecycleContext.java create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/StageLifecycles.java create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/StageType.java diff --git a/bigtop-manager-agent/pom.xml b/bigtop-manager-agent/pom.xml index e42589c9..192d66a1 100644 --- a/bigtop-manager-agent/pom.xml +++ b/bigtop-manager-agent/pom.xml @@ -97,6 +97,7 @@ + org.apache.maven.plugins maven-assembly-plugin diff --git a/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/message/type/pojo/ClusterInfo.java b/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/message/type/pojo/ClusterInfo.java index c5420173..208dbc28 100644 --- a/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/message/type/pojo/ClusterInfo.java +++ b/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/message/type/pojo/ClusterInfo.java @@ -3,7 +3,6 @@ package org.apache.bigtop.manager.common.message.type.pojo; import lombok.Data; import java.util.List; -import java.util.Set; @Data public class ClusterInfo { @@ -20,5 +19,5 @@ public class ClusterInfo { private String repoTemplate; - private Set packages; + private List packages; } diff --git a/bigtop-manager-server/pom.xml b/bigtop-manager-server/pom.xml index 9e6db746..3a6f79e9 100644 --- a/bigtop-manager-server/pom.xml +++ b/bigtop-manager-server/pom.xml @@ -195,6 +195,7 @@ + org.apache.maven.plugins maven-assembly-plugin diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/holder/SpringContextHolder.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/holder/SpringContextHolder.java index d54ee3ea..01b82fbc 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/holder/SpringContextHolder.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/holder/SpringContextHolder.java @@ -19,6 +19,7 @@ package org.apache.bigtop.manager.server.holder; import jakarta.annotation.Nonnull; import lombok.Getter; import org.apache.bigtop.manager.server.job.factory.JobFactory; +import org.apache.bigtop.manager.server.job.stage.StageLifecycle; import org.apache.bigtop.manager.server.job.validator.CommandValidator; import org.apache.bigtop.manager.server.ws.ServerWebSocketHandler; import org.springframework.context.ApplicationContext; @@ -49,4 +50,8 @@ public class SpringContextHolder implements ApplicationContextAware { public static Map getJobFactories() { return applicationContext.getBeansOfType(JobFactory.class); } + + public static Map getStageLifecycles() { + return applicationContext.getBeansOfType(StageLifecycle.class); + } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/AbstractJobFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/AbstractJobFactory.java index 75046ae4..f56ab7ee 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/AbstractJobFactory.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/AbstractJobFactory.java @@ -1,6 +1,7 @@ package org.apache.bigtop.manager.server.job.factory; import jakarta.annotation.Resource; +import org.apache.bigtop.manager.common.utils.JsonUtils; import org.apache.bigtop.manager.server.enums.JobState; import org.apache.bigtop.manager.server.orm.entity.Cluster; import org.apache.bigtop.manager.server.orm.entity.Job; @@ -59,6 +60,7 @@ public abstract class AbstractJobFactory implements JobFactory { job.setName(jobContext.getCommandDTO().getContext()); job.setState(JobState.PENDING); job.setCluster(cluster); + job.setPayload(JsonUtils.writeAsString(jobContext.getCommandDTO())); } protected void saveJob() { @@ -69,12 +71,14 @@ public abstract class AbstractJobFactory implements JobFactory { stage.setCluster(cluster); stage.setJob(job); stage.setStageOrder(i + 1); + stage.setState(JobState.PENDING); stageRepository.save(stage); for (Task task : stage.getTasks()) { task.setCluster(cluster); task.setJob(job); task.setStage(stage); + task.setState(JobState.PENDING); taskRepository.save(task); } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/cluster/ClusterCreateJobFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/cluster/ClusterCreateJobFactory.java index da721a27..37582485 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/cluster/ClusterCreateJobFactory.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/cluster/ClusterCreateJobFactory.java @@ -10,7 +10,6 @@ import org.apache.bigtop.manager.common.message.type.pojo.ComponentInfo; import org.apache.bigtop.manager.common.message.type.pojo.RepoInfo; import org.apache.bigtop.manager.common.utils.JsonUtils; import org.apache.bigtop.manager.server.enums.CommandLevel; -import org.apache.bigtop.manager.server.enums.JobState; import org.apache.bigtop.manager.server.job.CommandIdentifier; import org.apache.bigtop.manager.server.job.helper.HostCacheStageHelper; import org.apache.bigtop.manager.server.job.helper.HostCheckStageHelper; @@ -106,7 +105,6 @@ public class ClusterCreateJobFactory extends AbstractClusterJobFactory implement Stage hostCacheStage = new Stage(); hostCacheStage.setName(CACHE_STAGE_NAME); - hostCacheStage.setState(JobState.PENDING); hostCacheStage.setCallbackClassName(callbackClassName); hostCacheStage.setPayload(payload); @@ -123,7 +121,6 @@ public class ClusterCreateJobFactory extends AbstractClusterJobFactory implement task.setComponentName("bigtop-manager-agent"); task.setCommand(Command.CUSTOM_COMMAND); task.setCustomCommand("cache_host"); - task.setState(JobState.PENDING); RequestMessage requestMessage = hostCacheStageHelper.getMessage( hostname, diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/component/AbstractComponentJobFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/component/AbstractComponentJobFactory.java index 8e262eaa..cf1794f7 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/component/AbstractComponentJobFactory.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/component/AbstractComponentJobFactory.java @@ -122,7 +122,6 @@ public abstract class AbstractComponentJobFactory extends AbstractJobFactory { private Stage createStage(String callbackClassName, ComponentCommandWrapper componentCommandWrapper, Component component, Command command) { Stage stage = new Stage(); - stage.setState(JobState.PENDING); stage.setName(componentCommandWrapper.toDisplayString()); stage.setServiceName(component.getService().getServiceName()); stage.setComponentName(component.getComponentName()); @@ -163,9 +162,6 @@ public abstract class AbstractComponentJobFactory extends AbstractJobFactory { task.setCustomCommand(customCommand); task.setCommandScript(component.getCommandScript()); - // extra fields - task.setState(JobState.PENDING); - RequestMessage requestMessage = getMessage(component, hostname, command, customCommand); task.setContent(JsonUtils.writeAsString(requestMessage)); task.setMessageId(requestMessage.getMessageId()); diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/host/HostAddJobFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/host/HostAddJobFactory.java index 28b80c55..16ec07f3 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/host/HostAddJobFactory.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/host/HostAddJobFactory.java @@ -6,7 +6,6 @@ import org.apache.bigtop.manager.common.enums.Command; import org.apache.bigtop.manager.common.message.type.RequestMessage; import org.apache.bigtop.manager.common.utils.JsonUtils; import org.apache.bigtop.manager.server.enums.CommandLevel; -import org.apache.bigtop.manager.server.enums.JobState; import org.apache.bigtop.manager.server.job.CommandIdentifier; import org.apache.bigtop.manager.server.job.helper.HostCacheStageHelper; import org.apache.bigtop.manager.server.job.helper.HostCheckStageHelper; @@ -16,7 +15,7 @@ import org.apache.bigtop.manager.server.orm.entity.Cluster; import org.apache.bigtop.manager.server.orm.entity.Host; import org.apache.bigtop.manager.server.orm.entity.Stage; import org.apache.bigtop.manager.server.orm.entity.Task; -import org.apache.bigtop.manager.server.orm.repository.*; +import org.apache.bigtop.manager.server.orm.repository.HostRepository; import org.apache.bigtop.manager.server.service.HostService; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.config.ConfigurableBeanFactory; @@ -72,7 +71,6 @@ public class HostAddJobFactory extends AbstractHostJobFactory implements StageCa Stage hostCacheStage = new Stage(); hostCacheStage.setName(CACHE_STAGE_NAME); - hostCacheStage.setState(JobState.PENDING); if (StringUtils.isNotEmpty(callbackClassName)) { hostCacheStage.setCallbackClassName(callbackClassName); @@ -97,7 +95,6 @@ public class HostAddJobFactory extends AbstractHostJobFactory implements StageCa task.setComponentName("bigtop-manager-agent"); task.setCommand(Command.CUSTOM_COMMAND); task.setCustomCommand("cache_host"); - task.setState(JobState.PENDING); RequestMessage requestMessage = hostCheckStageHelper.createMessage(hostname); log.info("[HostCacheJobFactory-requestMessage]: {}", requestMessage); diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/service/ServiceInstallJobFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/service/ServiceInstallJobFactory.java index 00de83b9..d4842d39 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/service/ServiceInstallJobFactory.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/service/ServiceInstallJobFactory.java @@ -75,6 +75,11 @@ public class ServiceInstallJobFactory extends AbstractServiceJobFactory implemen // Check all required components in service stages.addAll(createStages(callbackClassName, Command.CHECK)); + // StageLifecycles.getStageLifecycle(StageType.COMPONENT_INSTALL).createStage(stageLifecycleContext); + // StageLifecycles.getStageLifecycle(StageType.COMPONENT_INSTALL).beforeStage(stageLifecycleContext); + // StageLifecycles.getStageLifecycle(StageType.COMPONENT_INSTALL).onStageSuccess(stageLifecycleContext); + // StageLifecycles.getStageLifecycle(StageType.COMPONENT_INSTALL).onStageFailed(stageLifecycleContext); + return stages; } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/helper/HostCacheStageHelper.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/helper/HostCacheStageHelper.java index c75d7000..cf6556cf 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/helper/HostCacheStageHelper.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/helper/HostCacheStageHelper.java @@ -63,7 +63,6 @@ public class HostCacheStageHelper { Stage hostCacheStage = new Stage(); hostCacheStage.setName(CACHE_STAGE_NAME); - hostCacheStage.setState(JobState.PENDING); if (StringUtils.isNotEmpty(callbackClassName)) { hostCacheStage.setCallbackClassName(callbackClassName); @@ -88,7 +87,6 @@ public class HostCacheStageHelper { task.setComponentName("bigtop-manager-agent"); task.setCommand(Command.CUSTOM_COMMAND); task.setCustomCommand("cache_host"); - task.setState(JobState.PENDING); RequestMessage requestMessage = getMessage(hostname); log.info("[HostCacheJobFactory-requestMessage]: {}", requestMessage); diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/helper/HostCheckStageHelper.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/helper/HostCheckStageHelper.java index 5c61dde3..478474c8 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/helper/HostCheckStageHelper.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/helper/HostCheckStageHelper.java @@ -39,7 +39,6 @@ public class HostCheckStageHelper { // Create stages Stage hostCheckStage = new Stage(); hostCheckStage.setName("Check Hosts"); - hostCheckStage.setState(JobState.PENDING); hostCheckStage.setCallbackClassName(callbackClassName); List tasks = new ArrayList<>(); @@ -55,7 +54,6 @@ public class HostCheckStageHelper { task.setComponentName("bigtop-manager-agent"); task.setCommand(Command.CUSTOM_COMMAND); task.setCustomCommand("check_host"); - task.setState(JobState.PENDING); RequestMessage requestMessage = createMessage(hostname); task.setContent(JsonUtils.writeAsString(requestMessage)); diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/listener/CommandEventListener.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/listener/CommandEventListener.java index b86f1012..42bae25f 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/listener/CommandEventListener.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/listener/CommandEventListener.java @@ -66,10 +66,9 @@ public class CommandEventListener { Boolean failed = asyncJobStrategy.handle(job, JobStrategyType.OVER_ON_FAIL); log.info("[CommandEventListener] failed: {}", failed); - // Reload job - job = jobRepository.getReferenceById(jobId); - if (!failed) { + // Reload job + job = jobRepository.getReferenceById(jobId); afterJobSuccess(job, commandDTO); } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/AbstractStageLifecycle.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/AbstractStageLifecycle.java new file mode 100644 index 00000000..f601b214 --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/AbstractStageLifecycle.java @@ -0,0 +1,24 @@ +package org.apache.bigtop.manager.server.job.stage; + +import lombok.Setter; + +@Setter +public abstract class AbstractStageLifecycle implements StageLifecycle { + + protected StageLifecycleContext context; + + @Override + public void beforeStage() { + + } + + @Override + public void onStageSuccess() { + + } + + @Override + public void onStageFailed() { + + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/ComponentInstallStageLifecycle.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/ComponentInstallStageLifecycle.java new file mode 100644 index 00000000..d30bde25 --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/ComponentInstallStageLifecycle.java @@ -0,0 +1,22 @@ +package org.apache.bigtop.manager.server.job.stage; + +import lombok.extern.slf4j.Slf4j; +import org.apache.bigtop.manager.server.orm.entity.Stage; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; + +@Slf4j +@org.springframework.stereotype.Component +@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) +public class ComponentInstallStageLifecycle extends AbstractStageLifecycle { + + @Override + public StageType getStageType() { + return StageType.COMPONENT_INSTALL; + } + + @Override + public Stage createStage() { + return null; + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/HostCacheStageLifecycle.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/HostCacheStageLifecycle.java new file mode 100644 index 00000000..d7605c2f --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/HostCacheStageLifecycle.java @@ -0,0 +1,253 @@ +package org.apache.bigtop.manager.server.job.stage; + +import com.fasterxml.jackson.core.type.TypeReference; +import jakarta.annotation.Resource; +import lombok.extern.slf4j.Slf4j; +import org.apache.bigtop.manager.common.constants.Constants; +import org.apache.bigtop.manager.common.enums.Command; +import org.apache.bigtop.manager.common.enums.MessageType; +import org.apache.bigtop.manager.common.message.type.HostCachePayload; +import org.apache.bigtop.manager.common.message.type.RequestMessage; +import org.apache.bigtop.manager.common.message.type.pojo.ClusterInfo; +import org.apache.bigtop.manager.common.message.type.pojo.ComponentInfo; +import org.apache.bigtop.manager.common.message.type.pojo.RepoInfo; +import org.apache.bigtop.manager.common.utils.JsonUtils; +import org.apache.bigtop.manager.server.model.dto.PropertyDTO; +import org.apache.bigtop.manager.server.model.dto.ServiceDTO; +import org.apache.bigtop.manager.server.model.dto.StackDTO; +import org.apache.bigtop.manager.server.model.mapper.RepoMapper; +import org.apache.bigtop.manager.server.orm.entity.*; +import org.apache.bigtop.manager.server.orm.repository.*; +import org.apache.bigtop.manager.server.utils.StackConfigUtils; +import org.apache.bigtop.manager.server.utils.StackUtils; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; + +import java.util.*; +import java.util.stream.Collectors; + +import static org.apache.bigtop.manager.common.constants.Constants.ALL_HOST_KEY; +import static org.apache.bigtop.manager.common.constants.Constants.CACHE_STAGE_NAME; + +@Slf4j +@org.springframework.stereotype.Component +@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) +public class HostCacheStageLifecycle extends AbstractStageLifecycle { + + @Resource + private ClusterRepository clusterRepository; + + @Resource + private HostComponentRepository hostComponentRepository; + + @Resource + private ServiceRepository serviceRepository; + + @Resource + private ServiceConfigMappingRepository serviceConfigMappingRepository; + + @Resource + private RepoRepository repoRepository; + + @Resource + private SettingRepository settingRepository; + + @Resource + private HostRepository hostRepository; + + @Resource + private ComponentRepository componentRepository; + + private ClusterInfo clusterInfo; + + private Map componentInfoMap; + + private Map> serviceConfigMap; + + private Map> hostMap; + + private List repoList; + + private Map> userMap; + + private Map settingsMap; + + @Override + public StageType getStageType() { + return StageType.HOST_CACHE; + } + + @Override + public Stage createStage() { + if (context.getClusterId() == null) { + genEmptyCaches(); + } else { + genCaches(); + } + + Stage hostCacheStage = new Stage(); + hostCacheStage.setName(CACHE_STAGE_NAME); + + List tasks = new ArrayList<>(); + for (String hostname : context.getHostnames()) { + Task task = new Task(); + task.setName("Cache host for " + hostname); + task.setStackName(context.getStackName()); + task.setStackVersion(context.getStackVersion()); + task.setHostname(hostname); + task.setServiceName("cluster"); + task.setServiceUser("root"); + task.setServiceGroup("root"); + task.setComponentName("bigtop-manager-agent"); + task.setCommand(Command.CUSTOM_COMMAND); + task.setCustomCommand("cache_host"); + + RequestMessage requestMessage = getMessage(hostname); + + log.info("[HostCacheJobFactory-requestMessage]: {}", requestMessage); + task.setContent(JsonUtils.writeAsString(requestMessage)); + task.setMessageId(requestMessage.getMessageId()); + tasks.add(task); + } + + hostCacheStage.setTasks(tasks); + return hostCacheStage; + } + + private void genCaches() { + Cluster cluster = clusterRepository.getReferenceById(context.getClusterId()); + + Long clusterId = cluster.getId(); + String clusterName = cluster.getClusterName(); + String stackName = cluster.getStack().getStackName(); + String stackVersion = cluster.getStack().getStackVersion(); + + List services = serviceRepository.findAllByClusterId(clusterId); + List serviceConfigMappingList = serviceConfigMappingRepository.findAllGroupLastest(clusterId); + List hostComponents = hostComponentRepository.findAllByComponentClusterId(clusterId); + List repos = repoRepository.findAllByCluster(cluster); + Iterable settings = settingRepository.findAll(); + List hostList = hostRepository.findAllByClusterId(clusterId); + + // Wrapper clusterInfo for HostCacheMessage + clusterInfo = new ClusterInfo(); + clusterInfo.setClusterName(clusterName); + clusterInfo.setStackName(stackName); + clusterInfo.setStackVersion(stackVersion); + clusterInfo.setUserGroup(cluster.getUserGroup()); + clusterInfo.setRepoTemplate(cluster.getRepoTemplate()); + clusterInfo.setRoot(cluster.getRoot()); + clusterInfo.setPackages(List.of(cluster.getPackages().split(","))); + + // Wrapper serviceConfigMap for HostCacheMessage + serviceConfigMap = new HashMap<>(); + serviceConfigMappingList.forEach(scm -> { + ServiceConfig sc = scm.getServiceConfig(); + List properties = JsonUtils.readFromString(sc.getPropertiesJson(), new TypeReference<>() { + }); + String configMapStr = JsonUtils.writeAsString(StackConfigUtils.extractConfigMap(properties)); + + if (serviceConfigMap.containsKey(sc.getService().getServiceName())) { + serviceConfigMap.get(sc.getService().getServiceName()).put(sc.getTypeName(), configMapStr); + } else { + Map hashMap = new HashMap<>(); + hashMap.put(sc.getTypeName(), configMapStr); + serviceConfigMap.put(sc.getService().getServiceName(), hashMap); + } + }); + + // Wrapper hostMap for HostCacheMessage + hostMap = new HashMap<>(); + hostComponents.forEach(x -> { + if (hostMap.containsKey(x.getComponent().getComponentName())) { + hostMap.get(x.getComponent().getComponentName()).add(x.getHost().getHostname()); + } else { + Set set = new HashSet<>(); + set.add(x.getHost().getHostname()); + hostMap.put(x.getComponent().getComponentName(), set); + } + hostMap.get(x.getComponent().getComponentName()).add(x.getHost().getHostname()); + }); + + Set hostNameSet = hostList.stream().map(Host::getHostname).collect(Collectors.toSet()); + hostMap.put(ALL_HOST_KEY, hostNameSet); + + // Wrapper repoList for HostCacheMessage + repoList = new ArrayList<>(); + repos.forEach(repo -> { + RepoInfo repoInfo = RepoMapper.INSTANCE.fromEntity2Message(repo); + repoList.add(repoInfo); + }); + + // Wrapper userMap for HostCacheMessage + userMap = new HashMap<>(); + services.forEach(x -> userMap.put(x.getServiceUser(), Set.of(x.getServiceGroup()))); + + // Wrapper settings for HostCacheMessage + settingsMap = new HashMap<>(); + settings.forEach(x -> settingsMap.put(x.getTypeName(), x.getConfigData())); + + // Wrapper componentInfoList for HostCacheMessage + componentInfoMap = new HashMap<>(); + List componentList = componentRepository.findAll(); + componentList.forEach(c -> { + ComponentInfo componentInfo = new ComponentInfo(); + componentInfo.setComponentName(c.getComponentName()); + componentInfo.setCommandScript(c.getCommandScript()); + componentInfo.setDisplayName(c.getDisplayName()); + componentInfo.setCategory(c.getCategory()); + componentInfo.setCustomCommands(c.getCustomCommands()); + componentInfo.setServiceName(c.getService().getServiceName()); + componentInfoMap.put(c.getComponentName(), componentInfo); + }); + } + + private void genEmptyCaches() { + componentInfoMap = new HashMap<>(); + serviceConfigMap = new HashMap<>(); + hostMap = new HashMap<>(); + userMap = new HashMap<>(); + settingsMap = new HashMap<>(); + + String fullStackName = StackUtils.fullStackName(context.getStackName(), context.getStackVersion()); + ImmutablePair> immutablePair = StackUtils.getStackKeyMap().get(fullStackName); + StackDTO stackDTO = immutablePair.getLeft(); + List serviceDTOList = immutablePair.getRight(); + + repoList = RepoMapper.INSTANCE.fromDTO2Message(context.getRepoInfoList()); + clusterInfo = new ClusterInfo(); + clusterInfo.setClusterName(context.getClusterName()); + clusterInfo.setStackName(context.getStackName()); + clusterInfo.setStackVersion(context.getStackVersion()); + clusterInfo.setUserGroup(stackDTO.getUserGroup()); + clusterInfo.setRepoTemplate(stackDTO.getRepoTemplate()); + clusterInfo.setRoot(stackDTO.getRoot()); + + List hostnames = context.getHostnames(); + hostMap.put(Constants.ALL_HOST_KEY, new HashSet<>(hostnames)); + + for (ServiceDTO serviceDTO : serviceDTOList) { + userMap.put(serviceDTO.getServiceUser(), Set.of(serviceDTO.getServiceGroup())); + } + } + + private RequestMessage getMessage(String hostname) { + HostCachePayload messagePayload = new HostCachePayload(); + messagePayload.setHostname(hostname); + messagePayload.setClusterInfo(clusterInfo); + messagePayload.setConfigurations(serviceConfigMap); + messagePayload.setClusterHostInfo(hostMap); + messagePayload.setRepoInfo(repoList); + messagePayload.setSettings(settingsMap); + messagePayload.setUserInfo(userMap); + messagePayload.setComponentInfo(componentInfoMap); + + RequestMessage requestMessage = new RequestMessage(); + requestMessage.setMessageType(MessageType.HOST_CACHE); + requestMessage.setHostname(hostname); + requestMessage.setMessagePayload(JsonUtils.writeAsString(messagePayload)); + + return requestMessage; + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/HostCheckStageLifecycle.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/HostCheckStageLifecycle.java new file mode 100644 index 00000000..290c96b8 --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/HostCheckStageLifecycle.java @@ -0,0 +1,97 @@ +package org.apache.bigtop.manager.server.job.stage; + +import com.fasterxml.jackson.core.type.TypeReference; +import jakarta.annotation.Resource; +import lombok.extern.slf4j.Slf4j; +import org.apache.bigtop.manager.common.constants.Constants; +import org.apache.bigtop.manager.common.enums.Command; +import org.apache.bigtop.manager.common.enums.MessageType; +import org.apache.bigtop.manager.common.message.type.HostCachePayload; +import org.apache.bigtop.manager.common.message.type.HostCheckPayload; +import org.apache.bigtop.manager.common.message.type.RequestMessage; +import org.apache.bigtop.manager.common.message.type.pojo.ClusterInfo; +import org.apache.bigtop.manager.common.message.type.pojo.ComponentInfo; +import org.apache.bigtop.manager.common.message.type.pojo.HostCheckType; +import org.apache.bigtop.manager.common.message.type.pojo.RepoInfo; +import org.apache.bigtop.manager.common.utils.JsonUtils; +import org.apache.bigtop.manager.server.model.dto.PropertyDTO; +import org.apache.bigtop.manager.server.model.dto.ServiceDTO; +import org.apache.bigtop.manager.server.model.dto.StackDTO; +import org.apache.bigtop.manager.server.model.mapper.RepoMapper; +import org.apache.bigtop.manager.server.orm.entity.*; +import org.apache.bigtop.manager.server.orm.repository.*; +import org.apache.bigtop.manager.server.utils.StackConfigUtils; +import org.apache.bigtop.manager.server.utils.StackUtils; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; + +import java.util.*; +import java.util.stream.Collectors; + +import static org.apache.bigtop.manager.common.constants.Constants.ALL_HOST_KEY; +import static org.apache.bigtop.manager.common.constants.Constants.CACHE_STAGE_NAME; + +@Slf4j +@org.springframework.stereotype.Component +@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) +public class HostCheckStageLifecycle extends AbstractStageLifecycle { + + @Resource + private ClusterRepository clusterRepository; + + public StageType getStageType() { + return StageType.HOST_CHECK; + } + + @Override + public Stage createStage() { + if (context.getClusterId() != null) { + Cluster cluster = clusterRepository.getReferenceById(context.getClusterId()); + + context.setStackName(cluster.getStack().getStackName()); + context.setStackVersion(cluster.getStack().getStackVersion()); + } + + // Create stages + Stage hostCheckStage = new Stage(); + hostCheckStage.setName("Check Hosts"); + + List tasks = new ArrayList<>(); + for (String hostname : context.getHostnames()) { + Task task = new Task(); + task.setName("Check host for " + hostname); + task.setStackName(context.getStackName()); + task.setStackVersion(context.getStackVersion()); + task.setHostname(hostname); + task.setServiceName("cluster"); + task.setServiceUser("root"); + task.setServiceGroup("root"); + task.setComponentName("bigtop-manager-agent"); + task.setCommand(Command.CUSTOM_COMMAND); + task.setCustomCommand("check_host"); + + RequestMessage requestMessage = createMessage(hostname); + task.setContent(JsonUtils.writeAsString(requestMessage)); + task.setMessageId(requestMessage.getMessageId()); + + tasks.add(task); + } + + hostCheckStage.setTasks(tasks); + return hostCheckStage; + } + + private RequestMessage createMessage(String hostname) { + HostCheckPayload messagePayload = new HostCheckPayload(); + messagePayload.setHostCheckTypes(HostCheckType.values()); + messagePayload.setHostname(hostname); + + RequestMessage requestMessage = new RequestMessage(); + requestMessage.setMessageType(MessageType.HOST_CHECK); + requestMessage.setHostname(hostname); + requestMessage.setMessagePayload(JsonUtils.writeAsString(messagePayload)); + + return requestMessage; + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/StageLifecycle.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/StageLifecycle.java new file mode 100644 index 00000000..598b4cd2 --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/StageLifecycle.java @@ -0,0 +1,40 @@ +package org.apache.bigtop.manager.server.job.stage; + +import org.apache.bigtop.manager.server.orm.entity.Stage; + +/** + * The StageLifecycle interface defines the lifecycle of a stage in a job. + * It provides methods for creating a stage, performing actions before the stage, + * handling successful completion of the stage, and handling failure of the stage. + */ +public interface StageLifecycle { + + /** + * Gets the type of the stage. + * + * @return the type of the stage + */ + StageType getStageType(); + + /** + * Creates a new Stage. + * + * @return a new Stage instance + */ + Stage createStage(); + + /** + * Performs actions before the stage starts. + */ + void beforeStage(); + + /** + * Handles the successful completion of the stage. + */ + void onStageSuccess(); + + /** + * Handles the failure of the stage. + */ + void onStageFailed(); +} \ No newline at end of file diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/StageLifecycleContext.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/StageLifecycleContext.java new file mode 100644 index 00000000..37722ef2 --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/StageLifecycleContext.java @@ -0,0 +1,26 @@ +package org.apache.bigtop.manager.server.job.stage; + +import lombok.Data; +import org.apache.bigtop.manager.server.model.dto.RepoDTO; + +import java.util.List; + +@Data +public class StageLifecycleContext { + + private Long clusterId; + + private String clusterName; + + private String stackName; + + private String stackVersion; + + private List hostnames; + + private String serviceName; + + private String componentName; + + private List repoInfoList; +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/StageLifecycles.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/StageLifecycles.java new file mode 100644 index 00000000..f1417a76 --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/StageLifecycles.java @@ -0,0 +1,45 @@ +package org.apache.bigtop.manager.server.job.stage; + +import lombok.extern.slf4j.Slf4j; +import org.apache.bigtop.manager.server.holder.SpringContextHolder; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +@Slf4j +public class StageLifecycles { + + private static final AtomicBoolean LOADED = new AtomicBoolean(false); + + private static final Map STAGE_LIFECYCLES = new HashMap<>(); + + public static StageLifecycle getStageLifecycle(StageType stageType) { + if (!LOADED.get()) { + load(); + } + + String beanName = STAGE_LIFECYCLES.get(stageType); + return SpringContextHolder.getApplicationContext().getBean(beanName, StageLifecycle.class); + } + + private static synchronized void load() { + if (LOADED.get()) { + return; + } + + for (Map.Entry entry : SpringContextHolder.getStageLifecycles().entrySet()) { + String beanName = entry.getKey(); + StageLifecycle stageLifecycle = entry.getValue(); + if (STAGE_LIFECYCLES.containsKey(stageLifecycle.getStageType())) { + log.error("Duplicate StageLifecycle with type: {}", stageLifecycle.getStageType()); + continue; + } + + STAGE_LIFECYCLES.put(stageLifecycle.getStageType(), beanName); + log.info("Load StageLifecycle: {} with type: {}", stageLifecycle.getClass().getName(), stageLifecycle.getStageType()); + } + + LOADED.set(true); + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/StageType.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/StageType.java new file mode 100644 index 00000000..fbb59588 --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/StageType.java @@ -0,0 +1,21 @@ +package org.apache.bigtop.manager.server.job.stage; + +public enum StageType { + + HOST_CHECK, + + HOST_CACHE, + + COMPONENT_INSTALL, + + COMPONENT_START, + + COMPONENT_STOP, + + COMPONENT_RESTART, + + SERVICE_CHECK, + + SERVICE_CONFIGURE, + ; +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/mapper/JobMapper.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/mapper/JobMapper.java index 4fe2401e..96677cb6 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/mapper/JobMapper.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/mapper/JobMapper.java @@ -1,13 +1,8 @@ package org.apache.bigtop.manager.server.model.mapper; -import org.apache.bigtop.manager.server.enums.JobState; -import org.apache.bigtop.manager.server.model.dto.CommandDTO; -import org.apache.bigtop.manager.server.model.dto.HostDTO; -import org.apache.bigtop.manager.server.model.vo.JobVO; import org.apache.bigtop.manager.server.model.vo.CommandVO; -import org.apache.bigtop.manager.server.orm.entity.Cluster; +import org.apache.bigtop.manager.server.model.vo.JobVO; import org.apache.bigtop.manager.server.orm.entity.Job; -import org.mapstruct.Context; import org.mapstruct.Mapper; import org.mapstruct.Mapping; import org.mapstruct.factory.Mappers; @@ -19,14 +14,6 @@ public interface JobMapper { JobMapper INSTANCE = Mappers.getMapper(JobMapper.class); - @Mapping(target = "cluster", expression = "java(cluster)") - @Mapping(target = "state", expression = "java(initState())") - Job fromDTO2Entity(CommandDTO commandDTO, @Context Cluster cluster); - - @Mapping(target = "cluster", expression = "java(cluster)") - @Mapping(target = "state", expression = "java(initState())") - Job fromDTO2Entity(HostDTO hostDTO, @Context Cluster cluster); - @Mapping(target = "createTime", dateFormat = "yyyy-MM-dd HH:mm:ss") @Mapping(target = "updateTime", dateFormat = "yyyy-MM-dd HH:mm:ss") JobVO fromEntity2VO(Job job); @@ -34,9 +21,4 @@ public interface JobMapper { List fromEntity2VO(List job); CommandVO fromEntity2CommandVO(Job job); - - default JobState initState() { - return JobState.PENDING; - } - } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/orm/entity/Job.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/orm/entity/Job.java index cf8fb240..943382c4 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/orm/entity/Job.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/orm/entity/Job.java @@ -27,6 +27,11 @@ public class Job extends BaseEntity { @Column(name = "name") private String name; + @Lob + @Basic(fetch = FetchType.LAZY) + @Column(name = "payload", length = 16777216) + private String payload; + @ManyToOne @JoinColumn(name = "cluster_id", foreignKey = @ForeignKey(ConstraintMode.NO_CONSTRAINT)) private Cluster cluster; diff --git a/bigtop-manager-ui/src/api/cluster/index.ts b/bigtop-manager-ui/src/api/cluster/index.ts index 4f19956c..de52ffb2 100644 --- a/bigtop-manager-ui/src/api/cluster/index.ts +++ b/bigtop-manager-ui/src/api/cluster/index.ts @@ -16,8 +16,7 @@ */ import request from '@/api/request.ts' -import { ClusterReq, ClusterVO } from '@/api/cluster/types.ts' -import { CommandVO } from '@/api/command/types.ts' +import { ClusterVO } from '@/api/cluster/types.ts' export const getClusters = (): Promise => { return request({ @@ -25,11 +24,3 @@ export const getClusters = (): Promise => { url: '/clusters' }) } - -export const createCluster = (data: ClusterReq): Promise => { - return request({ - method: 'post', - url: '/clusters', - data - }) -} -- Gitee From ee7a723960e35d3eeabb2a4a437224b364fa89bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E6=B2=BB=E5=9B=BD?= Date: Thu, 1 Feb 2024 11:59:24 +0800 Subject: [PATCH 2/4] 1 --- README.md | 1 + .../{job => command}/CommandIdentifier.java | 2 +- .../{job => command}/event/CommandEvent.java | 2 +- .../job/factory/AbstractJobFactory.java | 9 +- .../{ => command}/job/factory/JobContext.java | 2 +- .../job/factory/JobFactories.java | 4 +- .../{ => command}/job/factory/JobFactory.java | 4 +- .../cluster/AbstractClusterJobFactory.java | 4 +- .../cluster/ClusterCreateJobFactory.java} | 20 ++- .../AbstractComponentJobFactory.java | 18 +-- .../component/ComponentInstallJobFactory.java | 16 +- .../component/ComponentRestartJobFactory.java | 6 +- .../component/ComponentStartJobFactory.java | 6 +- .../component/ComponentStopJobFactory.java | 6 +- .../factory/host/AbstractHostJobFactory.java | 4 +- .../job/factory/host/HostAddJobFactory.java | 15 +- .../service/AbstractServiceJobFactory.java | 4 +- .../service/ServiceConfigureJobFactory.java | 14 +- .../service/ServiceInstallJobFactory.java | 17 +-- .../service/ServiceRestartJobFactory.java | 6 +- .../service/ServiceStartJobFactory.java | 6 +- .../service/ServiceStopJobFactory.java | 6 +- .../job/helper/HostCacheStageHelper.java | 2 +- .../job/helper/HostCheckStageHelper.java | 2 +- .../command/job/runner/AbstractJobRunner.java | 93 +++++++++++ .../server/command/job/runner/JobRunner.java | 19 +++ .../server/command/job/runner/JobRunners.java | 60 ++++++++ .../cluster/ClusterCreateJobRunner.java | 38 +++++ .../job/scheduler/DefaultJobScheduler.java | 53 +++++++ .../command/job/scheduler/JobScheduler.java | 28 ++++ .../job/validator/ClusterCreateValidator.java | 4 +- .../job/validator/ClusterHostValidator.java | 4 +- .../job/validator/CommandValidator.java | 4 +- .../job/validator/HostAddValidator.java | 4 +- .../validator/RequiredServicesValidator.java | 4 +- .../job/validator/ServiceHostValidator.java | 4 +- .../job/validator/StackValidator.java | 4 +- .../job/validator/ValidatorContext.java | 2 +- .../validator/ValidatorExecutionChain.java | 4 +- .../listener/CommandEventListener.java | 28 ++-- .../stage/factory/AbstractStageFactory.java | 26 ++++ .../command/stage/factory/StageContext.java | 75 +++++++++ .../stage/factory/StageFactories.java} | 24 +-- .../command/stage/factory/StageFactory.java | 24 +++ .../stage/factory}/StageType.java | 2 +- .../ComponentInstallStageFactory.java} | 10 +- .../factory/host/HostCacheStageFactory.java} | 14 +- .../factory/host/HostCheckStageFactory.java} | 39 ++--- .../stage/runner/AbstractStageRunner.java | 101 ++++++++++++ .../command/stage/runner/StageRunner.java | 19 +++ .../command/stage/runner/StageRunners.java | 55 +++++++ .../runner/host/HostCacheStageRunner.java} | 14 +- .../runner/host/HostCheckStageRunner.java | 17 +++ .../strategy/AbstractJobStrategy.java | 2 +- .../strategy/AsyncJobStrategy.java | 2 +- .../strategy/JobStrategy.java | 2 +- .../strategy/StageCallback.java | 2 +- .../strategy/SyncJobStrategy.java | 2 +- .../server/holder/SpringContextHolder.java | 20 ++- .../cluster/ClusterCreateJobFactory.java | 144 ------------------ .../job/stage/AbstractStageLifecycle.java | 24 --- .../server/job/stage/StageLifecycle.java | 40 ----- .../job/stage/StageLifecycleContext.java | 26 ---- .../manager/server/orm/entity/Stage.java | 5 + .../service/impl/CommandServiceImpl.java | 28 ++-- 65 files changed, 813 insertions(+), 433 deletions(-) rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{job => command}/CommandIdentifier.java (87%) rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{job => command}/event/CommandEvent.java (82%) rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{ => command}/job/factory/AbstractJobFactory.java (91%) rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{ => command}/job/factory/JobContext.java (77%) rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{ => command}/job/factory/JobFactories.java (93%) rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{ => command}/job/factory/JobFactory.java (58%) rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{ => command}/job/factory/cluster/AbstractClusterJobFactory.java (35%) rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{job/factory/component/ComponentCustomJobFactory.java => command/job/factory/cluster/ClusterCreateJobFactory.java} (32%) rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{ => command}/job/factory/component/AbstractComponentJobFactory.java (93%) rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{ => command}/job/factory/component/ComponentInstallJobFactory.java (84%) rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{ => command}/job/factory/component/ComponentRestartJobFactory.java (75%) rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{ => command}/job/factory/component/ComponentStartJobFactory.java (93%) rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{ => command}/job/factory/component/ComponentStopJobFactory.java (92%) rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{ => command}/job/factory/host/AbstractHostJobFactory.java (34%) rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{ => command}/job/factory/host/HostAddJobFactory.java (91%) rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{ => command}/job/factory/service/AbstractServiceJobFactory.java (92%) rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{ => command}/job/factory/service/ServiceConfigureJobFactory.java (82%) rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{ => command}/job/factory/service/ServiceInstallJobFactory.java (93%) rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{ => command}/job/factory/service/ServiceRestartJobFactory.java (75%) rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{ => command}/job/factory/service/ServiceStartJobFactory.java (91%) rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{ => command}/job/factory/service/ServiceStopJobFactory.java (91%) rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{ => command}/job/helper/HostCacheStageHelper.java (99%) rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{ => command}/job/helper/HostCheckStageHelper.java (98%) create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/AbstractJobRunner.java create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/JobRunner.java create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/JobRunners.java create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/cluster/ClusterCreateJobRunner.java create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/scheduler/DefaultJobScheduler.java create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/scheduler/JobScheduler.java rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{ => command}/job/validator/ClusterCreateValidator.java (91%) rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{ => command}/job/validator/ClusterHostValidator.java (91%) rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{ => command}/job/validator/CommandValidator.java (55%) rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{ => command}/job/validator/HostAddValidator.java (91%) rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{ => command}/job/validator/RequiredServicesValidator.java (96%) rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{ => command}/job/validator/ServiceHostValidator.java (92%) rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{ => command}/job/validator/StackValidator.java (91%) rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{ => command}/job/validator/ValidatorContext.java (74%) rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{ => command}/job/validator/ValidatorExecutionChain.java (81%) rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{job => command}/listener/CommandEventListener.java (82%) create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/AbstractStageFactory.java create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/StageContext.java rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{job/stage/StageLifecycles.java => command/stage/factory/StageFactories.java} (50%) create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/StageFactory.java rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{job/stage => command/stage/factory}/StageType.java (75%) rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{job/stage/ComponentInstallStageLifecycle.java => command/stage/factory/component/ComponentInstallStageFactory.java} (53%) rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{job/stage/HostCacheStageLifecycle.java => command/stage/factory/host/HostCacheStageFactory.java} (96%) rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{job/stage/HostCheckStageLifecycle.java => command/stage/factory/host/HostCheckStageFactory.java} (63%) create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/AbstractStageRunner.java create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/StageRunner.java create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/StageRunners.java rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{job/factory/service/ServiceCustomJobFactory.java => command/stage/runner/host/HostCacheStageRunner.java} (34%) create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/host/HostCheckStageRunner.java rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{job => command}/strategy/AbstractJobStrategy.java (93%) rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{job => command}/strategy/AsyncJobStrategy.java (99%) rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{job => command}/strategy/JobStrategy.java (78%) rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{job => command}/strategy/StageCallback.java (85%) rename bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/{job => command}/strategy/SyncJobStrategy.java (98%) delete mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/cluster/ClusterCreateJobFactory.java delete mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/AbstractStageLifecycle.java delete mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/StageLifecycle.java delete mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/StageLifecycleContext.java diff --git a/README.md b/README.md index 3e65bd37..f8114010 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ # Bigtop-Manager +Bigtop Manager 是一款现代化的，面向云原生的大数据集群管理平台，提供了一套轻量的方案来高效地管理 Bigtop 所支持的大数据组件 Bigtop-Manager is a platform for managing Bigtop components. Inspired by Apache Ambari. ## Prerequisites diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/CommandIdentifier.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/CommandIdentifier.java similarity index 87% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/CommandIdentifier.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/CommandIdentifier.java index 4b6e52eb..3960fd4c 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/CommandIdentifier.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/CommandIdentifier.java @@ -1,4 +1,4 @@ -package org.apache.bigtop.manager.server.job; +package org.apache.bigtop.manager.server.command; import lombok.AllArgsConstructor; import lombok.Data; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/event/CommandEvent.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/event/CommandEvent.java similarity index 82% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/event/CommandEvent.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/event/CommandEvent.java index 5683d0e9..f93923a2 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/event/CommandEvent.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/event/CommandEvent.java @@ -1,4 +1,4 @@ -package org.apache.bigtop.manager.server.job.event; +package org.apache.bigtop.manager.server.command.event; import lombok.Getter; import lombok.Setter; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/AbstractJobFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/AbstractJobFactory.java similarity index 91% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/AbstractJobFactory.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/AbstractJobFactory.java index f56ab7ee..8e389bb9 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/AbstractJobFactory.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/AbstractJobFactory.java @@ -1,4 +1,4 @@ -package org.apache.bigtop.manager.server.job.factory; +package org.apache.bigtop.manager.server.command.job.factory; import jakarta.annotation.Resource; import org.apache.bigtop.manager.common.utils.JsonUtils; @@ -12,6 +12,7 @@ import org.apache.bigtop.manager.server.orm.repository.JobRepository; import org.apache.bigtop.manager.server.orm.repository.StageRepository; import org.apache.bigtop.manager.server.orm.repository.TaskRepository; +import java.util.ArrayList; import java.util.List; public abstract class AbstractJobFactory implements JobFactory { @@ -34,6 +35,8 @@ public abstract class AbstractJobFactory implements JobFactory { protected Job job; + protected List stages = new ArrayList<>(); + @Override public Job createJob(JobContext jobContext) { this.jobContext = jobContext; @@ -42,7 +45,7 @@ public abstract class AbstractJobFactory implements JobFactory { initJob(); // Create stages and tasks for job - job.setStages(createStagesAndTasks()); + job.setStages(stages); // Save job saveJob(); @@ -50,7 +53,7 @@ public abstract class AbstractJobFactory implements JobFactory { return this.job; } - protected abstract List createStagesAndTasks(); + protected abstract void createStagesAndTasks(); private void initJob() { Long clusterId = jobContext.getCommandDTO().getClusterId(); diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/JobContext.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/JobContext.java similarity index 77% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/JobContext.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/JobContext.java index b603be65..da03c5c4 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/JobContext.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/JobContext.java @@ -1,4 +1,4 @@ -package org.apache.bigtop.manager.server.job.factory; +package org.apache.bigtop.manager.server.command.job.factory; import lombok.Data; import org.apache.bigtop.manager.server.model.dto.ClusterDTO; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/JobFactories.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/JobFactories.java similarity index 93% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/JobFactories.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/JobFactories.java index c520cc5f..276823bd 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/JobFactories.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/JobFactories.java @@ -1,10 +1,10 @@ -package org.apache.bigtop.manager.server.job.factory; +package org.apache.bigtop.manager.server.command.job.factory; import lombok.extern.slf4j.Slf4j; import org.apache.bigtop.manager.server.enums.ApiExceptionEnum; import org.apache.bigtop.manager.server.exception.ApiException; import org.apache.bigtop.manager.server.holder.SpringContextHolder; -import org.apache.bigtop.manager.server.job.CommandIdentifier; +import org.apache.bigtop.manager.server.command.CommandIdentifier; import java.util.HashMap; import java.util.Map; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/JobFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/JobFactory.java similarity index 58% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/JobFactory.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/JobFactory.java index 1283b1b8..a60557e7 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/JobFactory.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/JobFactory.java @@ -1,6 +1,6 @@ -package org.apache.bigtop.manager.server.job.factory; +package org.apache.bigtop.manager.server.command.job.factory; -import org.apache.bigtop.manager.server.job.CommandIdentifier; +import org.apache.bigtop.manager.server.command.CommandIdentifier; import org.apache.bigtop.manager.server.orm.entity.Job; public interface JobFactory { diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/cluster/AbstractClusterJobFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/cluster/AbstractClusterJobFactory.java similarity index 35% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/cluster/AbstractClusterJobFactory.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/cluster/AbstractClusterJobFactory.java index 7c50bebd..5747b791 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/cluster/AbstractClusterJobFactory.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/cluster/AbstractClusterJobFactory.java @@ -1,6 +1,6 @@ -package org.apache.bigtop.manager.server.job.factory.cluster; +package org.apache.bigtop.manager.server.command.job.factory.cluster; -import org.apache.bigtop.manager.server.job.factory.AbstractJobFactory; +import org.apache.bigtop.manager.server.command.job.factory.AbstractJobFactory; public abstract class AbstractClusterJobFactory extends AbstractJobFactory { diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/component/ComponentCustomJobFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/cluster/ClusterCreateJobFactory.java similarity index 32% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/component/ComponentCustomJobFactory.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/cluster/ClusterCreateJobFactory.java index 1e0da1e4..9dcfdd89 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/component/ComponentCustomJobFactory.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/cluster/ClusterCreateJobFactory.java @@ -1,20 +1,30 @@ -package org.apache.bigtop.manager.server.job.factory.component; +package org.apache.bigtop.manager.server.command.job.factory.cluster; import lombok.extern.slf4j.Slf4j; import org.apache.bigtop.manager.common.enums.Command; +import org.apache.bigtop.manager.common.utils.JsonUtils; +import org.apache.bigtop.manager.server.command.CommandIdentifier; +import org.apache.bigtop.manager.server.command.stage.factory.StageContext; +import org.apache.bigtop.manager.server.command.stage.factory.StageFactories; +import org.apache.bigtop.manager.server.command.stage.factory.StageType; import org.apache.bigtop.manager.server.enums.CommandLevel; -import org.apache.bigtop.manager.server.job.CommandIdentifier; -import org.apache.bigtop.manager.server.job.strategy.StageCallback; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Scope; @Slf4j @org.springframework.stereotype.Component @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) -public class ComponentCustomJobFactory extends AbstractComponentJobFactory implements StageCallback { +public class ClusterCreateJobFactory extends AbstractClusterJobFactory { @Override public CommandIdentifier getCommandIdentifier() { - return new CommandIdentifier(CommandLevel.COMPONENT, Command.CUSTOM_COMMAND); + return new CommandIdentifier(CommandLevel.CLUSTER, Command.CREATE); + } + + @Override + public void createStagesAndTasks() { + StageContext stageContext = StageContext.fromPayload(JsonUtils.writeAsString(jobContext.getCommandDTO())); + stages.add(StageFactories.getStageFactory(StageType.HOST_CHECK).createStage(stageContext)); + stages.add(StageFactories.getStageFactory(StageType.HOST_CACHE).createStage(stageContext)); } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/component/AbstractComponentJobFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/component/AbstractComponentJobFactory.java similarity index 93% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/component/AbstractComponentJobFactory.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/component/AbstractComponentJobFactory.java index cf1794f7..8d18f924 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/component/AbstractComponentJobFactory.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/component/AbstractComponentJobFactory.java @@ -1,4 +1,4 @@ -package org.apache.bigtop.manager.server.job.factory.component; +package org.apache.bigtop.manager.server.command.job.factory.component; import com.fasterxml.jackson.core.type.TypeReference; import jakarta.annotation.Resource; @@ -8,11 +8,10 @@ import org.apache.bigtop.manager.common.message.type.CommandPayload; import org.apache.bigtop.manager.common.message.type.RequestMessage; import org.apache.bigtop.manager.common.message.type.pojo.ScriptInfo; import org.apache.bigtop.manager.common.utils.JsonUtils; +import org.apache.bigtop.manager.server.command.job.factory.AbstractJobFactory; import org.apache.bigtop.manager.server.enums.ApiExceptionEnum; -import org.apache.bigtop.manager.server.enums.JobState; import org.apache.bigtop.manager.server.exception.ApiException; import org.apache.bigtop.manager.server.exception.ServerException; -import org.apache.bigtop.manager.server.job.factory.AbstractJobFactory; import org.apache.bigtop.manager.server.model.dto.CommandDTO; import org.apache.bigtop.manager.server.model.dto.command.ComponentCommandDTO; import org.apache.bigtop.manager.server.orm.entity.Component; @@ -39,22 +38,21 @@ public abstract class AbstractComponentJobFactory extends AbstractJobFactory { private ComponentRepository componentRepository; @Override - protected List createStagesAndTasks() { - return createStages(getClass().getName()); + protected void createStagesAndTasks() { + createStages(getClass().getName()); } - protected List createStages(String callbackClassName) { - return createStages(callbackClassName, jobContext.getCommandDTO().getCommand()); + protected void createStages(String callbackClassName) { + createStages(callbackClassName, jobContext.getCommandDTO().getCommand()); } - protected List createStages(String callbackClassName, Command command) { + protected void createStages(String callbackClassName, Command command) { CommandDTO commandDTO = jobContext.getCommandDTO(); String customCommand = commandDTO.getCustomCommand(); List sortedCommandWrappers = getSortedCommandWrappers(command); Map> componentHostMapping = getComponentHostMapping(command); - List stages = new ArrayList<>(); for (ComponentCommandWrapper componentCommandWrapper : sortedCommandWrappers) { String componentName = componentCommandWrapper.getComponentName(); Component component = componentCommandWrapper.getComponent(); @@ -75,8 +73,6 @@ public abstract class AbstractComponentJobFactory extends AbstractJobFactory { stage.setTasks(tasks); } - - return stages; } protected Map> getComponentHostMapping(Command command) { diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/component/ComponentInstallJobFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/component/ComponentInstallJobFactory.java similarity index 84% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/component/ComponentInstallJobFactory.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/component/ComponentInstallJobFactory.java index 069ac309..1b0528b0 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/component/ComponentInstallJobFactory.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/component/ComponentInstallJobFactory.java @@ -1,14 +1,14 @@ -package org.apache.bigtop.manager.server.job.factory.component; +package org.apache.bigtop.manager.server.command.job.factory.component; import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.apache.bigtop.manager.common.enums.Command; import org.apache.bigtop.manager.common.message.type.RequestMessage; import org.apache.bigtop.manager.common.utils.JsonUtils; +import org.apache.bigtop.manager.server.command.CommandIdentifier; +import org.apache.bigtop.manager.server.command.job.helper.HostCacheStageHelper; +import org.apache.bigtop.manager.server.command.strategy.StageCallback; import org.apache.bigtop.manager.server.enums.CommandLevel; -import org.apache.bigtop.manager.server.job.CommandIdentifier; -import org.apache.bigtop.manager.server.job.helper.HostCacheStageHelper; -import org.apache.bigtop.manager.server.job.strategy.StageCallback; import org.apache.bigtop.manager.server.model.dto.CommandDTO; import org.apache.bigtop.manager.server.orm.entity.Cluster; import org.apache.bigtop.manager.server.orm.entity.Stage; @@ -17,8 +17,6 @@ import org.apache.bigtop.manager.server.service.HostComponentService; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Scope; -import java.util.List; - import static org.apache.bigtop.manager.common.constants.Constants.CACHE_STAGE_NAME; @Slf4j @@ -38,14 +36,10 @@ public class ComponentInstallJobFactory extends AbstractComponentJobFactory impl } @Override - public List createStagesAndTasks() { - List stages = super.createStagesAndTasks(); - + public void createStagesAndTasks() { String callbackClassName = this.getClass().getName(); String payload = JsonUtils.writeAsString(jobContext.getCommandDTO()); stages.add(hostCacheStageHelper.createStage(cluster.getId(), callbackClassName, payload)); - - return stages; } @Override diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/component/ComponentRestartJobFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/component/ComponentRestartJobFactory.java similarity index 75% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/component/ComponentRestartJobFactory.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/component/ComponentRestartJobFactory.java index 7d00ead0..b3a65ff4 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/component/ComponentRestartJobFactory.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/component/ComponentRestartJobFactory.java @@ -1,10 +1,10 @@ -package org.apache.bigtop.manager.server.job.factory.component; +package org.apache.bigtop.manager.server.command.job.factory.component; import lombok.extern.slf4j.Slf4j; import org.apache.bigtop.manager.common.enums.Command; import org.apache.bigtop.manager.server.enums.CommandLevel; -import org.apache.bigtop.manager.server.job.CommandIdentifier; -import org.apache.bigtop.manager.server.job.strategy.StageCallback; +import org.apache.bigtop.manager.server.command.CommandIdentifier; +import org.apache.bigtop.manager.server.command.strategy.StageCallback; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Scope; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/component/ComponentStartJobFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/component/ComponentStartJobFactory.java similarity index 93% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/component/ComponentStartJobFactory.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/component/ComponentStartJobFactory.java index 5f6064c9..5815d121 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/component/ComponentStartJobFactory.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/component/ComponentStartJobFactory.java @@ -1,4 +1,4 @@ -package org.apache.bigtop.manager.server.job.factory.component; +package org.apache.bigtop.manager.server.command.job.factory.component; import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; @@ -7,8 +7,8 @@ import org.apache.bigtop.manager.common.utils.JsonUtils; import org.apache.bigtop.manager.server.enums.CommandLevel; import org.apache.bigtop.manager.server.enums.JobState; import org.apache.bigtop.manager.server.enums.MaintainState; -import org.apache.bigtop.manager.server.job.CommandIdentifier; -import org.apache.bigtop.manager.server.job.strategy.StageCallback; +import org.apache.bigtop.manager.server.command.CommandIdentifier; +import org.apache.bigtop.manager.server.command.strategy.StageCallback; import org.apache.bigtop.manager.server.model.dto.CommandDTO; import org.apache.bigtop.manager.server.model.dto.command.ComponentCommandDTO; import org.apache.bigtop.manager.server.orm.entity.HostComponent; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/component/ComponentStopJobFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/component/ComponentStopJobFactory.java similarity index 92% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/component/ComponentStopJobFactory.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/component/ComponentStopJobFactory.java index 48ce3421..5e6363ac 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/component/ComponentStopJobFactory.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/component/ComponentStopJobFactory.java @@ -1,4 +1,4 @@ -package org.apache.bigtop.manager.server.job.factory.component; +package org.apache.bigtop.manager.server.command.job.factory.component; import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; @@ -7,8 +7,8 @@ import org.apache.bigtop.manager.common.utils.JsonUtils; import org.apache.bigtop.manager.server.enums.CommandLevel; import org.apache.bigtop.manager.server.enums.JobState; import org.apache.bigtop.manager.server.enums.MaintainState; -import org.apache.bigtop.manager.server.job.CommandIdentifier; -import org.apache.bigtop.manager.server.job.strategy.StageCallback; +import org.apache.bigtop.manager.server.command.CommandIdentifier; +import org.apache.bigtop.manager.server.command.strategy.StageCallback; import org.apache.bigtop.manager.server.model.dto.CommandDTO; import org.apache.bigtop.manager.server.model.dto.command.ComponentCommandDTO; import org.apache.bigtop.manager.server.orm.entity.*; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/host/AbstractHostJobFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/host/AbstractHostJobFactory.java similarity index 34% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/host/AbstractHostJobFactory.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/host/AbstractHostJobFactory.java index d8e5eece..20e49eb9 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/host/AbstractHostJobFactory.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/host/AbstractHostJobFactory.java @@ -1,6 +1,6 @@ -package org.apache.bigtop.manager.server.job.factory.host; +package org.apache.bigtop.manager.server.command.job.factory.host; -import org.apache.bigtop.manager.server.job.factory.AbstractJobFactory; +import org.apache.bigtop.manager.server.command.job.factory.AbstractJobFactory; public abstract class AbstractHostJobFactory extends AbstractJobFactory { diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/host/HostAddJobFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/host/HostAddJobFactory.java similarity index 91% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/host/HostAddJobFactory.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/host/HostAddJobFactory.java index 16ec07f3..34a21a6f 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/host/HostAddJobFactory.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/host/HostAddJobFactory.java @@ -1,4 +1,4 @@ -package org.apache.bigtop.manager.server.job.factory.host; +package org.apache.bigtop.manager.server.command.job.factory.host; import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; @@ -6,10 +6,10 @@ import org.apache.bigtop.manager.common.enums.Command; import org.apache.bigtop.manager.common.message.type.RequestMessage; import org.apache.bigtop.manager.common.utils.JsonUtils; import org.apache.bigtop.manager.server.enums.CommandLevel; -import org.apache.bigtop.manager.server.job.CommandIdentifier; -import org.apache.bigtop.manager.server.job.helper.HostCacheStageHelper; -import org.apache.bigtop.manager.server.job.helper.HostCheckStageHelper; -import org.apache.bigtop.manager.server.job.strategy.StageCallback; +import org.apache.bigtop.manager.server.command.CommandIdentifier; +import org.apache.bigtop.manager.server.command.job.helper.HostCacheStageHelper; +import org.apache.bigtop.manager.server.command.job.helper.HostCheckStageHelper; +import org.apache.bigtop.manager.server.command.strategy.StageCallback; import org.apache.bigtop.manager.server.model.dto.command.HostCommandDTO; import org.apache.bigtop.manager.server.orm.entity.Cluster; import org.apache.bigtop.manager.server.orm.entity.Host; @@ -49,8 +49,7 @@ public class HostAddJobFactory extends AbstractHostJobFactory implements StageCa } @Override - public List createStagesAndTasks() { - List stages = new ArrayList<>(); + public void createStagesAndTasks() { String callbackClassName = this.getClass().getName(); Long clusterId = jobContext.getCommandDTO().getClusterId(); List hostnames = jobContext.getCommandDTO().getHostCommands().stream().map(HostCommandDTO::getHostname).toList(); @@ -58,8 +57,6 @@ public class HostAddJobFactory extends AbstractHostJobFactory implements StageCa // Create stages stages.add(hostCheckStageHelper.createStage(clusterId, hostnames, callbackClassName)); stages.add(createStage(callbackClassName, JsonUtils.writeAsString(hostnames))); - - return stages; } public Stage createStage(String callbackClassName, String payload) { diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/service/AbstractServiceJobFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/AbstractServiceJobFactory.java similarity index 92% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/service/AbstractServiceJobFactory.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/AbstractServiceJobFactory.java index 81cfb71f..c7a0d380 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/service/AbstractServiceJobFactory.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/AbstractServiceJobFactory.java @@ -1,8 +1,8 @@ -package org.apache.bigtop.manager.server.job.factory.service; +package org.apache.bigtop.manager.server.command.job.factory.service; import jakarta.annotation.Resource; import org.apache.bigtop.manager.common.enums.Command; -import org.apache.bigtop.manager.server.job.factory.component.AbstractComponentJobFactory; +import org.apache.bigtop.manager.server.command.job.factory.component.AbstractComponentJobFactory; import org.apache.bigtop.manager.server.model.dto.command.ServiceCommandDTO; import org.apache.bigtop.manager.server.orm.entity.Component; import org.apache.bigtop.manager.server.orm.entity.HostComponent; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/service/ServiceConfigureJobFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/ServiceConfigureJobFactory.java similarity index 82% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/service/ServiceConfigureJobFactory.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/ServiceConfigureJobFactory.java index 8918fe02..f33316ce 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/service/ServiceConfigureJobFactory.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/ServiceConfigureJobFactory.java @@ -1,13 +1,13 @@ -package org.apache.bigtop.manager.server.job.factory.service; +package org.apache.bigtop.manager.server.command.job.factory.service; import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.apache.bigtop.manager.common.enums.Command; import org.apache.bigtop.manager.common.utils.JsonUtils; +import org.apache.bigtop.manager.server.command.CommandIdentifier; +import org.apache.bigtop.manager.server.command.job.helper.HostCacheStageHelper; +import org.apache.bigtop.manager.server.command.strategy.StageCallback; import org.apache.bigtop.manager.server.enums.CommandLevel; -import org.apache.bigtop.manager.server.job.CommandIdentifier; -import org.apache.bigtop.manager.server.job.helper.HostCacheStageHelper; -import org.apache.bigtop.manager.server.job.strategy.StageCallback; import org.apache.bigtop.manager.server.model.dto.CommandDTO; import org.apache.bigtop.manager.server.model.dto.ServiceConfigDTO; import org.apache.bigtop.manager.server.model.dto.command.ServiceCommandDTO; @@ -19,8 +19,6 @@ import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Scope; -import java.util.List; - @Slf4j @org.springframework.stereotype.Component @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) @@ -41,10 +39,10 @@ public class ServiceConfigureJobFactory extends AbstractServiceJobFactory implem } @Override - public List createStagesAndTasks() { + public void createStagesAndTasks() { String callbackClassName = this.getClass().getName(); String payload = JsonUtils.writeAsString(jobContext.getCommandDTO()); - return List.of(hostCacheStageHelper.createStage(cluster.getId(), callbackClassName, payload)); + stages.add(hostCacheStageHelper.createStage(cluster.getId(), callbackClassName, payload)); } @Override diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/service/ServiceInstallJobFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/ServiceInstallJobFactory.java similarity index 93% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/service/ServiceInstallJobFactory.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/ServiceInstallJobFactory.java index d4842d39..dba957a7 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/service/ServiceInstallJobFactory.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/ServiceInstallJobFactory.java @@ -1,4 +1,4 @@ -package org.apache.bigtop.manager.server.job.factory.service; +package org.apache.bigtop.manager.server.command.job.factory.service; import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; @@ -8,9 +8,9 @@ import org.apache.bigtop.manager.common.utils.JsonUtils; import org.apache.bigtop.manager.server.enums.CommandLevel; import org.apache.bigtop.manager.server.enums.JobState; import org.apache.bigtop.manager.server.enums.MaintainState; -import org.apache.bigtop.manager.server.job.CommandIdentifier; -import org.apache.bigtop.manager.server.job.helper.HostCacheStageHelper; -import org.apache.bigtop.manager.server.job.strategy.StageCallback; +import org.apache.bigtop.manager.server.command.CommandIdentifier; +import org.apache.bigtop.manager.server.command.job.helper.HostCacheStageHelper; +import org.apache.bigtop.manager.server.command.strategy.StageCallback; import org.apache.bigtop.manager.server.model.dto.*; import org.apache.bigtop.manager.server.model.dto.command.ServiceCommandDTO; import org.apache.bigtop.manager.server.model.mapper.ComponentMapper; @@ -59,9 +59,9 @@ public class ServiceInstallJobFactory extends AbstractServiceJobFactory implemen * create job and persist it to database */ @Override - protected List createStagesAndTasks() { + protected void createStagesAndTasks() { // Install all required components in service - List stages = super.createStagesAndTasks(); + super.createStagesAndTasks(); String callbackClassName = this.getClass().getName(); @@ -70,17 +70,16 @@ public class ServiceInstallJobFactory extends AbstractServiceJobFactory implemen stages.add(hostCacheStageHelper.createStage(cluster.getId(), callbackClassName, payload)); // Start all required components in service - stages.addAll(createStages(callbackClassName, Command.START)); + createStages(callbackClassName, Command.START); // Check all required components in service - stages.addAll(createStages(callbackClassName, Command.CHECK)); + createStages(callbackClassName, Command.CHECK); // StageLifecycles.getStageLifecycle(StageType.COMPONENT_INSTALL).createStage(stageLifecycleContext); // StageLifecycles.getStageLifecycle(StageType.COMPONENT_INSTALL).beforeStage(stageLifecycleContext); // StageLifecycles.getStageLifecycle(StageType.COMPONENT_INSTALL).onStageSuccess(stageLifecycleContext); // StageLifecycles.getStageLifecycle(StageType.COMPONENT_INSTALL).onStageFailed(stageLifecycleContext); - return stages; } @Override diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/service/ServiceRestartJobFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/ServiceRestartJobFactory.java similarity index 75% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/service/ServiceRestartJobFactory.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/ServiceRestartJobFactory.java index 4f296543..1160b842 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/service/ServiceRestartJobFactory.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/ServiceRestartJobFactory.java @@ -1,10 +1,10 @@ -package org.apache.bigtop.manager.server.job.factory.service; +package org.apache.bigtop.manager.server.command.job.factory.service; import lombok.extern.slf4j.Slf4j; import org.apache.bigtop.manager.common.enums.Command; import org.apache.bigtop.manager.server.enums.CommandLevel; -import org.apache.bigtop.manager.server.job.CommandIdentifier; -import org.apache.bigtop.manager.server.job.strategy.StageCallback; +import org.apache.bigtop.manager.server.command.CommandIdentifier; +import org.apache.bigtop.manager.server.command.strategy.StageCallback; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Scope; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/service/ServiceStartJobFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/ServiceStartJobFactory.java similarity index 91% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/service/ServiceStartJobFactory.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/ServiceStartJobFactory.java index a6ea7e1f..c9e80fb2 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/service/ServiceStartJobFactory.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/ServiceStartJobFactory.java @@ -1,4 +1,4 @@ -package org.apache.bigtop.manager.server.job.factory.service; +package org.apache.bigtop.manager.server.command.job.factory.service; import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; @@ -6,8 +6,8 @@ import org.apache.bigtop.manager.common.enums.Command; import org.apache.bigtop.manager.server.enums.CommandLevel; import org.apache.bigtop.manager.server.enums.JobState; import org.apache.bigtop.manager.server.enums.MaintainState; -import org.apache.bigtop.manager.server.job.CommandIdentifier; -import org.apache.bigtop.manager.server.job.strategy.StageCallback; +import org.apache.bigtop.manager.server.command.CommandIdentifier; +import org.apache.bigtop.manager.server.command.strategy.StageCallback; import org.apache.bigtop.manager.server.orm.entity.HostComponent; import org.apache.bigtop.manager.server.orm.entity.Service; import org.apache.bigtop.manager.server.orm.entity.Stage; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/service/ServiceStopJobFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/ServiceStopJobFactory.java similarity index 91% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/service/ServiceStopJobFactory.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/ServiceStopJobFactory.java index 0a212460..1592edb6 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/service/ServiceStopJobFactory.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/ServiceStopJobFactory.java @@ -1,4 +1,4 @@ -package org.apache.bigtop.manager.server.job.factory.service; +package org.apache.bigtop.manager.server.command.job.factory.service; import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; @@ -6,8 +6,8 @@ import org.apache.bigtop.manager.common.enums.Command; import org.apache.bigtop.manager.server.enums.CommandLevel; import org.apache.bigtop.manager.server.enums.JobState; import org.apache.bigtop.manager.server.enums.MaintainState; -import org.apache.bigtop.manager.server.job.CommandIdentifier; -import org.apache.bigtop.manager.server.job.strategy.StageCallback; +import org.apache.bigtop.manager.server.command.CommandIdentifier; +import org.apache.bigtop.manager.server.command.strategy.StageCallback; import org.apache.bigtop.manager.server.orm.entity.HostComponent; import org.apache.bigtop.manager.server.orm.entity.Service; import org.apache.bigtop.manager.server.orm.entity.Stage; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/helper/HostCacheStageHelper.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/helper/HostCacheStageHelper.java similarity index 99% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/helper/HostCacheStageHelper.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/helper/HostCacheStageHelper.java index cf6556cf..ad68666f 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/helper/HostCacheStageHelper.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/helper/HostCacheStageHelper.java @@ -1,4 +1,4 @@ -package org.apache.bigtop.manager.server.job.helper; +package org.apache.bigtop.manager.server.command.job.helper; import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.collect.Sets; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/helper/HostCheckStageHelper.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/helper/HostCheckStageHelper.java similarity index 98% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/helper/HostCheckStageHelper.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/helper/HostCheckStageHelper.java index 478474c8..b248f2d1 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/helper/HostCheckStageHelper.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/helper/HostCheckStageHelper.java @@ -1,4 +1,4 @@ -package org.apache.bigtop.manager.server.job.helper; +package org.apache.bigtop.manager.server.command.job.helper; import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/AbstractJobRunner.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/AbstractJobRunner.java new file mode 100644 index 00000000..721db9cb --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/AbstractJobRunner.java @@ -0,0 +1,93 @@ +package org.apache.bigtop.manager.server.command.job.runner; + +import jakarta.annotation.Resource; +import org.apache.bigtop.manager.server.command.stage.runner.StageRunner; +import org.apache.bigtop.manager.server.command.stage.runner.StageRunners; +import org.apache.bigtop.manager.server.enums.JobState; +import org.apache.bigtop.manager.server.orm.entity.Job; +import org.apache.bigtop.manager.server.orm.entity.Stage; +import org.apache.bigtop.manager.server.orm.entity.Task; +import org.apache.bigtop.manager.server.orm.repository.JobRepository; +import org.apache.bigtop.manager.server.orm.repository.StageRepository; +import org.apache.bigtop.manager.server.orm.repository.TaskRepository; + +import java.util.Comparator; +import java.util.List; +import java.util.concurrent.LinkedBlockingQueue; + +public abstract class AbstractJobRunner implements JobRunner { + + @Resource + private JobRepository jobRepository; + + @Resource + private StageRepository stageRepository; + + @Resource + private TaskRepository taskRepository; + + protected Job job; + + @Override + public void setJob(Job job) { + this.job = job; + } + + @Override + public void run() { + beforeRun(); + + // Sort stage + List stages = job.getStages(); + stages.sort(Comparator.comparingInt(Stage::getStageOrder)); + + boolean success = true; + LinkedBlockingQueue queue = new LinkedBlockingQueue<>(stages); + while (!queue.isEmpty()) { + Stage stage = queue.poll(); + StageRunner runner = StageRunners.getStageRunner(stage); + runner.run(); + + if (stage.getState() == JobState.FAILED) { + success = false; + break; + } + } + + if (success) { + onSuccess(); + } else { + onFailure(); + } + } + + @Override + public void beforeRun() { + job.setState(JobState.PROCESSING); + jobRepository.save(job); + } + + @Override + public void onSuccess() { + job.setState(JobState.SUCCESSFUL); + jobRepository.save(job); + } + + @Override + public void onFailure() { + for (Stage stage : job.getStages()) { + if (stage.getState() == JobState.PENDING) { + stage.setState(JobState.CANCELED); + stageRepository.save(stage); + + for (Task task : stage.getTasks()) { + task.setState(JobState.CANCELED); + taskRepository.save(task); + } + } + } + + job.setState(JobState.FAILED); + jobRepository.save(job); + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/JobRunner.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/JobRunner.java new file mode 100644 index 00000000..f18a5906 --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/JobRunner.java @@ -0,0 +1,19 @@ +package org.apache.bigtop.manager.server.command.job.runner; + +import org.apache.bigtop.manager.server.command.CommandIdentifier; +import org.apache.bigtop.manager.server.orm.entity.Job; + +public interface JobRunner { + + CommandIdentifier getCommandIdentifier(); + + void setJob(Job job); + + void beforeRun(); + + void run(); + + void onSuccess(); + + void onFailure(); +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/JobRunners.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/JobRunners.java new file mode 100644 index 00000000..9b52a365 --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/JobRunners.java @@ -0,0 +1,60 @@ +package org.apache.bigtop.manager.server.command.job.runner; + +import lombok.extern.slf4j.Slf4j; +import org.apache.bigtop.manager.common.utils.JsonUtils; +import org.apache.bigtop.manager.server.command.CommandIdentifier; +import org.apache.bigtop.manager.server.enums.ApiExceptionEnum; +import org.apache.bigtop.manager.server.exception.ApiException; +import org.apache.bigtop.manager.server.holder.SpringContextHolder; +import org.apache.bigtop.manager.server.model.dto.CommandDTO; +import org.apache.bigtop.manager.server.orm.entity.Job; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +@Slf4j +public class JobRunners { + + private static final AtomicBoolean LOADED = new AtomicBoolean(false); + + private static final Map JOB_RUNNERS = new HashMap<>(); + + public static JobRunner getJobRunner(Job job) { + if (!LOADED.get()) { + load(); + } + + CommandDTO commandDTO = JsonUtils.readFromString(job.getPayload(), CommandDTO.class); + CommandIdentifier identifier = new CommandIdentifier(commandDTO.getCommandLevel(), commandDTO.getCommand()); + if (!JOB_RUNNERS.containsKey(identifier)) { + throw new ApiException(ApiExceptionEnum.COMMAND_NOT_SUPPORTED, identifier.getCommand().toLowerCase(), identifier.getCommandLevel().toLowerCase()); + } + + String beanName = JOB_RUNNERS.get(identifier); + JobRunner runner = SpringContextHolder.getApplicationContext().getBean(beanName, JobRunner.class); + runner.setJob(job); + + return runner; + } + + private static synchronized void load() { + if (LOADED.get()) { + return; + } + + for (Map.Entry entry : SpringContextHolder.getJobRunners().entrySet()) { + String beanName = entry.getKey(); + JobRunner jobRunner = entry.getValue(); + if (JOB_RUNNERS.containsKey(jobRunner.getCommandIdentifier())) { + log.error("Duplicate JobRunner with identifier: {}", jobRunner.getCommandIdentifier()); + continue; + } + + JOB_RUNNERS.put(jobRunner.getCommandIdentifier(), beanName); + log.info("Load JobRunner: {} with identifier: {}", jobRunner.getClass().getName(), jobRunner.getCommandIdentifier()); + } + + LOADED.set(true); + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/cluster/ClusterCreateJobRunner.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/cluster/ClusterCreateJobRunner.java new file mode 100644 index 00000000..a89919ed --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/cluster/ClusterCreateJobRunner.java @@ -0,0 +1,38 @@ +package org.apache.bigtop.manager.server.command.job.runner.cluster; + +import jakarta.annotation.Resource; +import lombok.extern.slf4j.Slf4j; +import org.apache.bigtop.manager.common.enums.Command; +import org.apache.bigtop.manager.common.utils.JsonUtils; +import org.apache.bigtop.manager.server.command.CommandIdentifier; +import org.apache.bigtop.manager.server.command.job.runner.AbstractJobRunner; +import org.apache.bigtop.manager.server.enums.CommandLevel; +import org.apache.bigtop.manager.server.model.dto.ClusterDTO; +import org.apache.bigtop.manager.server.model.dto.command.ClusterCommandDTO; +import org.apache.bigtop.manager.server.service.ClusterService; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; + +@Slf4j +@org.springframework.stereotype.Component +@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) +public class ClusterCreateJobRunner extends AbstractJobRunner { + + @Resource + private ClusterService clusterService; + + @Override + public CommandIdentifier getCommandIdentifier() { + return new CommandIdentifier(CommandLevel.CLUSTER, Command.CREATE); + } + + @Override + public void onSuccess() { + ClusterCommandDTO clusterCommand = JsonUtils.readFromString(job.getPayload(), ClusterCommandDTO.class); + ClusterDTO clusterDTO = new ClusterDTO(); + BeanUtils.copyProperties(clusterCommand, clusterDTO); + + clusterService.save(clusterDTO); + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/scheduler/DefaultJobScheduler.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/scheduler/DefaultJobScheduler.java new file mode 100644 index 00000000..d05a9cf4 --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/scheduler/DefaultJobScheduler.java @@ -0,0 +1,53 @@ +package org.apache.bigtop.manager.server.command.job.scheduler; + +import jakarta.annotation.PostConstruct; +import lombok.extern.slf4j.Slf4j; +import org.apache.bigtop.manager.server.command.job.runner.JobRunner; +import org.apache.bigtop.manager.server.command.job.runner.JobRunners; +import org.apache.bigtop.manager.server.orm.entity.Job; +import org.springframework.stereotype.Component; + +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; + +@Slf4j +@Component +public class DefaultJobScheduler implements JobScheduler { + + private final LinkedBlockingQueue queue = new LinkedBlockingQueue<>(); + + private final Executor executor = Executors.newSingleThreadExecutor(); + + private volatile boolean running = true; + + @Override + public void submit(Job job) { + queue.offer(job); + } + + @Override + public void start() { + running = true; + } + + @Override + public void stop() { + running = false; + } + + @PostConstruct + public void init() { + executor.execute(() -> { + while (running) { + try { + Job job = queue.take(); + JobRunner runner = JobRunners.getJobRunner(job); + runner.run(); + } catch (InterruptedException e) { + log.warn("Error when polling new job", e); + } + } + }); + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/scheduler/JobScheduler.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/scheduler/JobScheduler.java new file mode 100644 index 00000000..6443c8f0 --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/scheduler/JobScheduler.java @@ -0,0 +1,28 @@ +package org.apache.bigtop.manager.server.command.job.scheduler; + +import org.apache.bigtop.manager.server.orm.entity.Job; + +/** + * JobScheduler interface for job management. + * This interface provides methods to submit, start, and stop jobs. + */ +public interface JobScheduler { + + /** + * Submits a job to the job scheduler. + * @param job The job to be submitted. + */ + void submit(Job job); + + /** + * Starts the job scheduler. + * This method should be called after all jobs have been submitted. + */ + void start(); + + /** + * Stops the job scheduler. + * This method should be called to gracefully stop the job scheduler. + */ + void stop(); +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/validator/ClusterCreateValidator.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/validator/ClusterCreateValidator.java similarity index 91% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/validator/ClusterCreateValidator.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/validator/ClusterCreateValidator.java index 457e7b26..93b6ff5a 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/validator/ClusterCreateValidator.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/validator/ClusterCreateValidator.java @@ -1,11 +1,11 @@ -package org.apache.bigtop.manager.server.job.validator; +package org.apache.bigtop.manager.server.command.job.validator; import jakarta.annotation.Resource; import org.apache.bigtop.manager.common.enums.Command; import org.apache.bigtop.manager.server.enums.ApiExceptionEnum; import org.apache.bigtop.manager.server.enums.CommandLevel; import org.apache.bigtop.manager.server.exception.ApiException; -import org.apache.bigtop.manager.server.job.CommandIdentifier; +import org.apache.bigtop.manager.server.command.CommandIdentifier; import org.apache.bigtop.manager.server.model.dto.command.ClusterCommandDTO; import org.apache.bigtop.manager.server.orm.entity.Cluster; import org.apache.bigtop.manager.server.orm.repository.ClusterRepository; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/validator/ClusterHostValidator.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/validator/ClusterHostValidator.java similarity index 91% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/validator/ClusterHostValidator.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/validator/ClusterHostValidator.java index 7e64ceda..4fae245a 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/validator/ClusterHostValidator.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/validator/ClusterHostValidator.java @@ -1,11 +1,11 @@ -package org.apache.bigtop.manager.server.job.validator; +package org.apache.bigtop.manager.server.command.job.validator; import jakarta.annotation.Resource; import org.apache.bigtop.manager.common.enums.Command; import org.apache.bigtop.manager.server.enums.ApiExceptionEnum; import org.apache.bigtop.manager.server.enums.CommandLevel; import org.apache.bigtop.manager.server.exception.ApiException; -import org.apache.bigtop.manager.server.job.CommandIdentifier; +import org.apache.bigtop.manager.server.command.CommandIdentifier; import org.apache.bigtop.manager.server.model.dto.command.ClusterCommandDTO; import org.apache.bigtop.manager.server.orm.entity.Host; import org.apache.bigtop.manager.server.orm.repository.HostRepository; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/validator/CommandValidator.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/validator/CommandValidator.java similarity index 55% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/validator/CommandValidator.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/validator/CommandValidator.java index 993a4445..43d9d4f6 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/validator/CommandValidator.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/validator/CommandValidator.java @@ -1,6 +1,6 @@ -package org.apache.bigtop.manager.server.job.validator; +package org.apache.bigtop.manager.server.command.job.validator; -import org.apache.bigtop.manager.server.job.CommandIdentifier; +import org.apache.bigtop.manager.server.command.CommandIdentifier; import java.util.List; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/validator/HostAddValidator.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/validator/HostAddValidator.java similarity index 91% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/validator/HostAddValidator.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/validator/HostAddValidator.java index e0e10ca3..9a935c44 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/validator/HostAddValidator.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/validator/HostAddValidator.java @@ -1,11 +1,11 @@ -package org.apache.bigtop.manager.server.job.validator; +package org.apache.bigtop.manager.server.command.job.validator; import jakarta.annotation.Resource; import org.apache.bigtop.manager.common.enums.Command; import org.apache.bigtop.manager.server.enums.ApiExceptionEnum; import org.apache.bigtop.manager.server.enums.CommandLevel; import org.apache.bigtop.manager.server.exception.ApiException; -import org.apache.bigtop.manager.server.job.CommandIdentifier; +import org.apache.bigtop.manager.server.command.CommandIdentifier; import org.apache.bigtop.manager.server.model.dto.command.HostCommandDTO; import org.apache.bigtop.manager.server.orm.entity.Host; import org.apache.bigtop.manager.server.orm.repository.HostRepository; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/validator/RequiredServicesValidator.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/validator/RequiredServicesValidator.java similarity index 96% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/validator/RequiredServicesValidator.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/validator/RequiredServicesValidator.java index a79ce475..ac7515aa 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/validator/RequiredServicesValidator.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/validator/RequiredServicesValidator.java @@ -1,11 +1,11 @@ -package org.apache.bigtop.manager.server.job.validator; +package org.apache.bigtop.manager.server.command.job.validator; import jakarta.annotation.Resource; import org.apache.bigtop.manager.common.enums.Command; import org.apache.bigtop.manager.server.enums.ApiExceptionEnum; import org.apache.bigtop.manager.server.enums.CommandLevel; import org.apache.bigtop.manager.server.exception.ApiException; -import org.apache.bigtop.manager.server.job.CommandIdentifier; +import org.apache.bigtop.manager.server.command.CommandIdentifier; import org.apache.bigtop.manager.server.model.dto.CommandDTO; import org.apache.bigtop.manager.server.model.dto.ServiceDTO; import org.apache.bigtop.manager.server.model.dto.StackDTO; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/validator/ServiceHostValidator.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/validator/ServiceHostValidator.java similarity index 92% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/validator/ServiceHostValidator.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/validator/ServiceHostValidator.java index ac6a5686..8be1dcd0 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/validator/ServiceHostValidator.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/validator/ServiceHostValidator.java @@ -1,11 +1,11 @@ -package org.apache.bigtop.manager.server.job.validator; +package org.apache.bigtop.manager.server.command.job.validator; import jakarta.annotation.Resource; import org.apache.bigtop.manager.common.enums.Command; import org.apache.bigtop.manager.server.enums.ApiExceptionEnum; import org.apache.bigtop.manager.server.enums.CommandLevel; import org.apache.bigtop.manager.server.exception.ApiException; -import org.apache.bigtop.manager.server.job.CommandIdentifier; +import org.apache.bigtop.manager.server.command.CommandIdentifier; import org.apache.bigtop.manager.server.model.dto.CommandDTO; import org.apache.bigtop.manager.server.model.dto.command.ServiceCommandDTO; import org.apache.bigtop.manager.server.orm.entity.Host; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/validator/StackValidator.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/validator/StackValidator.java similarity index 91% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/validator/StackValidator.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/validator/StackValidator.java index c80482e7..2e704ba8 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/validator/StackValidator.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/validator/StackValidator.java @@ -1,11 +1,11 @@ -package org.apache.bigtop.manager.server.job.validator; +package org.apache.bigtop.manager.server.command.job.validator; import jakarta.annotation.Resource; import org.apache.bigtop.manager.common.enums.Command; import org.apache.bigtop.manager.server.enums.ApiExceptionEnum; import org.apache.bigtop.manager.server.enums.CommandLevel; import org.apache.bigtop.manager.server.exception.ApiException; -import org.apache.bigtop.manager.server.job.CommandIdentifier; +import org.apache.bigtop.manager.server.command.CommandIdentifier; import org.apache.bigtop.manager.server.model.dto.command.ClusterCommandDTO; import org.apache.bigtop.manager.server.orm.entity.Stack; import org.apache.bigtop.manager.server.orm.repository.StackRepository; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/validator/ValidatorContext.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/validator/ValidatorContext.java similarity index 74% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/validator/ValidatorContext.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/validator/ValidatorContext.java index 0bd4c532..6f2f90f6 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/validator/ValidatorContext.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/validator/ValidatorContext.java @@ -1,4 +1,4 @@ -package org.apache.bigtop.manager.server.job.validator; +package org.apache.bigtop.manager.server.command.job.validator; import lombok.Data; import org.apache.bigtop.manager.server.model.dto.CommandDTO; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/validator/ValidatorExecutionChain.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/validator/ValidatorExecutionChain.java similarity index 81% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/validator/ValidatorExecutionChain.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/validator/ValidatorExecutionChain.java index 89ee04b4..305b7c6e 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/validator/ValidatorExecutionChain.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/validator/ValidatorExecutionChain.java @@ -1,7 +1,7 @@ -package org.apache.bigtop.manager.server.job.validator; +package org.apache.bigtop.manager.server.command.job.validator; import org.apache.bigtop.manager.server.holder.SpringContextHolder; -import org.apache.bigtop.manager.server.job.CommandIdentifier; +import org.apache.bigtop.manager.server.command.CommandIdentifier; import java.util.List; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/listener/CommandEventListener.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/listener/CommandEventListener.java similarity index 82% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/listener/CommandEventListener.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/listener/CommandEventListener.java index 42bae25f..44ce6322 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/listener/CommandEventListener.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/listener/CommandEventListener.java @@ -14,15 +14,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.bigtop.manager.server.job.listener; +package org.apache.bigtop.manager.server.command.listener; import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.apache.bigtop.manager.common.enums.Command; +import org.apache.bigtop.manager.server.command.event.CommandEvent; +import org.apache.bigtop.manager.server.command.job.scheduler.JobScheduler; +import org.apache.bigtop.manager.server.command.strategy.AsyncJobStrategy; import org.apache.bigtop.manager.server.enums.CommandLevel; -import org.apache.bigtop.manager.server.enums.JobStrategyType; -import org.apache.bigtop.manager.server.job.event.CommandEvent; -import org.apache.bigtop.manager.server.job.strategy.AsyncJobStrategy; import org.apache.bigtop.manager.server.model.dto.CommandDTO; import org.apache.bigtop.manager.server.orm.entity.Cluster; import org.apache.bigtop.manager.server.orm.entity.Job; @@ -52,6 +52,9 @@ public class CommandEventListener { @Resource private ClusterRepository clusterRepository; + @Resource + private JobScheduler jobScheduler; + @Resource private AsyncJobStrategy asyncJobStrategy; @@ -63,14 +66,15 @@ public class CommandEventListener { Long jobId = event.getJobId(); Job job = jobRepository.getReferenceById(jobId); - Boolean failed = asyncJobStrategy.handle(job, JobStrategyType.OVER_ON_FAIL); - log.info("[CommandEventListener] failed: {}", failed); - - if (!failed) { - // Reload job - job = jobRepository.getReferenceById(jobId); - afterJobSuccess(job, commandDTO); - } + jobScheduler.submit(job); +// Boolean failed = asyncJobStrategy.handle(job, JobStrategyType.OVER_ON_FAIL); +// log.info("[CommandEventListener] failed: {}", failed); +// +// if (!failed) { +// // Reload job +// job = jobRepository.getReferenceById(jobId); +// afterJobSuccess(job, commandDTO); +// } } private void afterJobSuccess(Job job, CommandDTO commandDTO) { diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/AbstractStageFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/AbstractStageFactory.java new file mode 100644 index 00000000..1c25fd57 --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/AbstractStageFactory.java @@ -0,0 +1,26 @@ +package org.apache.bigtop.manager.server.command.stage.factory; + +import org.apache.bigtop.manager.common.utils.JsonUtils; +import org.apache.bigtop.manager.server.orm.entity.Stage; + +public abstract class AbstractStageFactory implements StageFactory { + + protected StageContext context; + + protected Stage stage; + + @Override + public Stage createStage(StageContext context) { + this.context = context; + this.context.setStageType(getStageType()); + + this.stage = new Stage(); + this.stage.setContext(JsonUtils.writeAsString(context)); + + doCreateStage(); + + return this.stage; + } + + protected abstract void doCreateStage(); +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/StageContext.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/StageContext.java new file mode 100644 index 00000000..b361b57b --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/StageContext.java @@ -0,0 +1,75 @@ +package org.apache.bigtop.manager.server.command.stage.factory; + +import lombok.Data; +import org.apache.bigtop.manager.common.enums.Command; +import org.apache.bigtop.manager.common.utils.JsonUtils; +import org.apache.bigtop.manager.server.enums.CommandLevel; +import org.apache.bigtop.manager.server.model.dto.CommandDTO; +import org.apache.bigtop.manager.server.model.dto.RepoDTO; +import org.apache.bigtop.manager.server.model.dto.command.ClusterCommandDTO; +import org.apache.bigtop.manager.server.model.dto.command.HostCommandDTO; + +import java.util.List; + +@Data +public class StageContext { + + private StageType stageType; + + private Long clusterId; + + private String clusterName; + + private String stackName; + + private String stackVersion; + + private List hostnames; + + private String serviceName; + + private String componentName; + + private List repoInfoList; + + public static StageContext fromPayload(String payload) { + StageContext context = new StageContext(); + CommandDTO commandDTO = JsonUtils.readFromString(payload, CommandDTO.class); + + context.setClusterId(commandDTO.getClusterId()); + + switch (commandDTO.getCommandLevel()) { + case CLUSTER -> fromClusterCommandPayload(context, commandDTO); + case HOST -> fromHostCommandPayload(context, commandDTO); + case SERVICE -> fromServiceCommandPayload(context, commandDTO); + case COMPONENT -> fromComponentCommandPayload(context, commandDTO); + } + + return context; + } + + private static void fromClusterCommandPayload(StageContext context, CommandDTO commandDTO) { + ClusterCommandDTO clusterCommand = commandDTO.getClusterCommand(); + + context.setClusterName(clusterCommand.getClusterName()); + context.setStackName(clusterCommand.getStackName()); + context.setStackVersion(clusterCommand.getStackVersion()); + context.setHostnames(clusterCommand.getHostnames()); + context.setRepoInfoList(clusterCommand.getRepoInfoList()); + } + + private static void fromHostCommandPayload(StageContext context, CommandDTO commandDTO) { + List hostCommands = commandDTO.getHostCommands(); + + List hostnames = hostCommands.stream().map(HostCommandDTO::getHostname).toList(); + context.setHostnames(hostnames); + } + + private static void fromServiceCommandPayload(StageContext context, CommandDTO commandDTO) { + // No need to set anything here, we should deal with this in the service job factory + } + + private static void fromComponentCommandPayload(StageContext context, CommandDTO commandDTO) { + // No need to set anything here, we should deal with this in the component job factory + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/StageLifecycles.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/StageFactories.java similarity index 50% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/StageLifecycles.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/StageFactories.java index f1417a76..e3803025 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/StageLifecycles.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/StageFactories.java @@ -1,4 +1,4 @@ -package org.apache.bigtop.manager.server.job.stage; +package org.apache.bigtop.manager.server.command.stage.factory; import lombok.extern.slf4j.Slf4j; import org.apache.bigtop.manager.server.holder.SpringContextHolder; @@ -8,19 +8,19 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; @Slf4j -public class StageLifecycles { +public class StageFactories { private static final AtomicBoolean LOADED = new AtomicBoolean(false); - private static final Map STAGE_LIFECYCLES = new HashMap<>(); + private static final Map STAGE_FACTORIES = new HashMap<>(); - public static StageLifecycle getStageLifecycle(StageType stageType) { + public static StageFactory getStageFactory(StageType stageType) { if (!LOADED.get()) { load(); } - String beanName = STAGE_LIFECYCLES.get(stageType); - return SpringContextHolder.getApplicationContext().getBean(beanName, StageLifecycle.class); + String beanName = STAGE_FACTORIES.get(stageType); + return SpringContextHolder.getApplicationContext().getBean(beanName, StageFactory.class); } private static synchronized void load() { @@ -28,16 +28,16 @@ public class StageLifecycles { return; } - for (Map.Entry entry : SpringContextHolder.getStageLifecycles().entrySet()) { + for (Map.Entry entry : SpringContextHolder.getStageFactories().entrySet()) { String beanName = entry.getKey(); - StageLifecycle stageLifecycle = entry.getValue(); - if (STAGE_LIFECYCLES.containsKey(stageLifecycle.getStageType())) { - log.error("Duplicate StageLifecycle with type: {}", stageLifecycle.getStageType()); + StageFactory stageFactory = entry.getValue(); + if (STAGE_FACTORIES.containsKey(stageFactory.getStageType())) { + log.error("Duplicate StageLifecycle with type: {}", stageFactory.getStageType()); continue; } - STAGE_LIFECYCLES.put(stageLifecycle.getStageType(), beanName); - log.info("Load StageLifecycle: {} with type: {}", stageLifecycle.getClass().getName(), stageLifecycle.getStageType()); + STAGE_FACTORIES.put(stageFactory.getStageType(), beanName); + log.info("Load StageLifecycle: {} with type: {}", stageFactory.getClass().getName(), stageFactory.getStageType()); } LOADED.set(true); diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/StageFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/StageFactory.java new file mode 100644 index 00000000..0a3b35ad --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/StageFactory.java @@ -0,0 +1,24 @@ +package org.apache.bigtop.manager.server.command.stage.factory; + +import org.apache.bigtop.manager.server.orm.entity.Stage; + +/** + * Stage factory. + */ +public interface StageFactory { + + /** + * Gets the type of the stage. + * + * @return the type of the stage + */ + StageType getStageType(); + + /** + * Creates a stage. + * + * @param context the stage context + * @return the stage + */ + Stage createStage(StageContext context); +} \ No newline at end of file diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/StageType.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/StageType.java similarity index 75% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/StageType.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/StageType.java index fbb59588..df6e6c8b 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/StageType.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/StageType.java @@ -1,4 +1,4 @@ -package org.apache.bigtop.manager.server.job.stage; +package org.apache.bigtop.manager.server.command.stage.factory; public enum StageType { diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/ComponentInstallStageLifecycle.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/component/ComponentInstallStageFactory.java similarity index 53% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/ComponentInstallStageLifecycle.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/component/ComponentInstallStageFactory.java index d30bde25..0bdcbc22 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/ComponentInstallStageLifecycle.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/component/ComponentInstallStageFactory.java @@ -1,14 +1,15 @@ -package org.apache.bigtop.manager.server.job.stage; +package org.apache.bigtop.manager.server.command.stage.factory.component; import lombok.extern.slf4j.Slf4j; -import org.apache.bigtop.manager.server.orm.entity.Stage; +import org.apache.bigtop.manager.server.command.stage.factory.AbstractStageFactory; +import org.apache.bigtop.manager.server.command.stage.factory.StageType; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Scope; @Slf4j @org.springframework.stereotype.Component @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) -public class ComponentInstallStageLifecycle extends AbstractStageLifecycle { +public class ComponentInstallStageFactory extends AbstractStageFactory { @Override public StageType getStageType() { @@ -16,7 +17,6 @@ public class ComponentInstallStageLifecycle extends AbstractStageLifecycle { } @Override - public Stage createStage() { - return null; + public void doCreateStage() { } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/HostCacheStageLifecycle.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/host/HostCacheStageFactory.java similarity index 96% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/HostCacheStageLifecycle.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/host/HostCacheStageFactory.java index d7605c2f..da061348 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/HostCacheStageLifecycle.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/host/HostCacheStageFactory.java @@ -1,4 +1,4 @@ -package org.apache.bigtop.manager.server.job.stage; +package org.apache.bigtop.manager.server.command.stage.factory.host; import com.fasterxml.jackson.core.type.TypeReference; import jakarta.annotation.Resource; @@ -12,6 +12,8 @@ import org.apache.bigtop.manager.common.message.type.pojo.ClusterInfo; import org.apache.bigtop.manager.common.message.type.pojo.ComponentInfo; import org.apache.bigtop.manager.common.message.type.pojo.RepoInfo; import org.apache.bigtop.manager.common.utils.JsonUtils; +import org.apache.bigtop.manager.server.command.stage.factory.AbstractStageFactory; +import org.apache.bigtop.manager.server.command.stage.factory.StageType; import org.apache.bigtop.manager.server.model.dto.PropertyDTO; import org.apache.bigtop.manager.server.model.dto.ServiceDTO; import org.apache.bigtop.manager.server.model.dto.StackDTO; @@ -33,7 +35,7 @@ import static org.apache.bigtop.manager.common.constants.Constants.CACHE_STAGE_N @Slf4j @org.springframework.stereotype.Component @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) -public class HostCacheStageLifecycle extends AbstractStageLifecycle { +public class HostCacheStageFactory extends AbstractStageFactory { @Resource private ClusterRepository clusterRepository; @@ -79,15 +81,14 @@ public class HostCacheStageLifecycle extends AbstractStageLifecycle { } @Override - public Stage createStage() { + public void doCreateStage() { if (context.getClusterId() == null) { genEmptyCaches(); } else { genCaches(); } - Stage hostCacheStage = new Stage(); - hostCacheStage.setName(CACHE_STAGE_NAME); + stage.setName(CACHE_STAGE_NAME); List tasks = new ArrayList<>(); for (String hostname : context.getHostnames()) { @@ -111,8 +112,7 @@ public class HostCacheStageLifecycle extends AbstractStageLifecycle { tasks.add(task); } - hostCacheStage.setTasks(tasks); - return hostCacheStage; + stage.setTasks(tasks); } private void genCaches() { diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/HostCheckStageLifecycle.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/host/HostCheckStageFactory.java similarity index 63% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/HostCheckStageLifecycle.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/host/HostCheckStageFactory.java index 290c96b8..3af2dddc 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/HostCheckStageLifecycle.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/host/HostCheckStageFactory.java @@ -1,41 +1,28 @@ -package org.apache.bigtop.manager.server.job.stage; +package org.apache.bigtop.manager.server.command.stage.factory.host; -import com.fasterxml.jackson.core.type.TypeReference; import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; -import org.apache.bigtop.manager.common.constants.Constants; import org.apache.bigtop.manager.common.enums.Command; import org.apache.bigtop.manager.common.enums.MessageType; -import org.apache.bigtop.manager.common.message.type.HostCachePayload; import org.apache.bigtop.manager.common.message.type.HostCheckPayload; import org.apache.bigtop.manager.common.message.type.RequestMessage; -import org.apache.bigtop.manager.common.message.type.pojo.ClusterInfo; -import org.apache.bigtop.manager.common.message.type.pojo.ComponentInfo; import org.apache.bigtop.manager.common.message.type.pojo.HostCheckType; -import org.apache.bigtop.manager.common.message.type.pojo.RepoInfo; import org.apache.bigtop.manager.common.utils.JsonUtils; -import org.apache.bigtop.manager.server.model.dto.PropertyDTO; -import org.apache.bigtop.manager.server.model.dto.ServiceDTO; -import org.apache.bigtop.manager.server.model.dto.StackDTO; -import org.apache.bigtop.manager.server.model.mapper.RepoMapper; -import org.apache.bigtop.manager.server.orm.entity.*; -import org.apache.bigtop.manager.server.orm.repository.*; -import org.apache.bigtop.manager.server.utils.StackConfigUtils; -import org.apache.bigtop.manager.server.utils.StackUtils; -import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.bigtop.manager.server.command.stage.factory.AbstractStageFactory; +import org.apache.bigtop.manager.server.command.stage.factory.StageType; +import org.apache.bigtop.manager.server.orm.entity.Cluster; +import org.apache.bigtop.manager.server.orm.entity.Task; +import org.apache.bigtop.manager.server.orm.repository.ClusterRepository; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Scope; -import java.util.*; -import java.util.stream.Collectors; - -import static org.apache.bigtop.manager.common.constants.Constants.ALL_HOST_KEY; -import static org.apache.bigtop.manager.common.constants.Constants.CACHE_STAGE_NAME; +import java.util.ArrayList; +import java.util.List; @Slf4j @org.springframework.stereotype.Component @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) -public class HostCheckStageLifecycle extends AbstractStageLifecycle { +public class HostCheckStageFactory extends AbstractStageFactory { @Resource private ClusterRepository clusterRepository; @@ -45,7 +32,7 @@ public class HostCheckStageLifecycle extends AbstractStageLifecycle { } @Override - public Stage createStage() { + public void doCreateStage() { if (context.getClusterId() != null) { Cluster cluster = clusterRepository.getReferenceById(context.getClusterId()); @@ -54,8 +41,7 @@ public class HostCheckStageLifecycle extends AbstractStageLifecycle { } // Create stages - Stage hostCheckStage = new Stage(); - hostCheckStage.setName("Check Hosts"); + stage.setName("Check Hosts"); List tasks = new ArrayList<>(); for (String hostname : context.getHostnames()) { @@ -78,8 +64,7 @@ public class HostCheckStageLifecycle extends AbstractStageLifecycle { tasks.add(task); } - hostCheckStage.setTasks(tasks); - return hostCheckStage; + stage.setTasks(tasks); } private RequestMessage createMessage(String hostname) { diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/AbstractStageRunner.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/AbstractStageRunner.java new file mode 100644 index 00000000..a94cdb9a --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/AbstractStageRunner.java @@ -0,0 +1,101 @@ +package org.apache.bigtop.manager.server.command.stage.runner; + +import jakarta.annotation.Resource; +import lombok.extern.slf4j.Slf4j; +import org.apache.bigtop.manager.common.constants.MessageConstants; +import org.apache.bigtop.manager.common.message.type.BaseCommandMessage; +import org.apache.bigtop.manager.common.message.type.RequestMessage; +import org.apache.bigtop.manager.common.message.type.ResultMessage; +import org.apache.bigtop.manager.common.utils.JsonUtils; +import org.apache.bigtop.manager.server.enums.JobState; +import org.apache.bigtop.manager.server.holder.SpringContextHolder; +import org.apache.bigtop.manager.server.orm.entity.Stage; +import org.apache.bigtop.manager.server.orm.entity.Task; +import org.apache.bigtop.manager.server.orm.repository.StageRepository; +import org.apache.bigtop.manager.server.orm.repository.TaskRepository; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; + +import static org.apache.bigtop.manager.common.constants.Constants.COMMAND_MESSAGE_RESPONSE_TIMEOUT; + +@Slf4j +public abstract class AbstractStageRunner implements StageRunner { + + @Resource + private StageRepository stageRepository; + + @Resource + private TaskRepository taskRepository; + + protected Stage stage; + + @Override + public void setStage(Stage stage) { + this.stage = stage; + } + + @Override + public void run() { + beforeRun(); + + List> futures = new ArrayList<>(); + for (Task task : stage.getTasks()) { + task.setState(JobState.PROCESSING); + taskRepository.save(task); + + BaseCommandMessage message = JsonUtils.readFromString(task.getContent(), RequestMessage.class); + message.setTaskId(task.getId()); + message.setStageId(stage.getId()); + message.setJobId(stage.getJob().getId()); + + futures.add(CompletableFuture.supplyAsync(() -> { + ResultMessage res = SpringContextHolder.getServerWebSocket().sendMessage(task.getHostname(), message); + + log.info("Execute task {} completed: {}", task.getId(), res); + boolean taskSuccess = res.getCode() == MessageConstants.SUCCESS_CODE; + + task.setState(taskSuccess ? JobState.SUCCESSFUL : JobState.FAILED); + taskRepository.save(task); + + return taskSuccess; + })); + } + + List taskResults = futures.stream().map((future) -> { + try { + return future.get(COMMAND_MESSAGE_RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS); + } catch (Exception e) { + log.error("Error running task", e); + return false; + } + }).toList(); + + boolean allTaskSuccess = taskResults.stream().allMatch(Boolean::booleanValue); + if (allTaskSuccess) { + onSuccess(); + } else { + onFailure(); + } + } + + @Override + public void beforeRun() { + stage.setState(JobState.PROCESSING); + stageRepository.save(stage); + } + + @Override + public void onSuccess() { + stage.setState(JobState.SUCCESSFUL); + stageRepository.save(stage); + } + + @Override + public void onFailure() { + stage.setState(JobState.FAILED); + stageRepository.save(stage); + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/StageRunner.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/StageRunner.java new file mode 100644 index 00000000..75c98093 --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/StageRunner.java @@ -0,0 +1,19 @@ +package org.apache.bigtop.manager.server.command.stage.runner; + +import org.apache.bigtop.manager.server.command.stage.factory.StageType; +import org.apache.bigtop.manager.server.orm.entity.Stage; + +public interface StageRunner { + + StageType getStageType(); + + void setStage(Stage stage); + + void beforeRun(); + + void run(); + + void onSuccess(); + + void onFailure(); +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/StageRunners.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/StageRunners.java new file mode 100644 index 00000000..201da569 --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/StageRunners.java @@ -0,0 +1,55 @@ +package org.apache.bigtop.manager.server.command.stage.runner; + +import lombok.extern.slf4j.Slf4j; +import org.apache.bigtop.manager.common.utils.JsonUtils; +import org.apache.bigtop.manager.server.command.stage.factory.StageContext; +import org.apache.bigtop.manager.server.command.stage.factory.StageType; +import org.apache.bigtop.manager.server.holder.SpringContextHolder; +import org.apache.bigtop.manager.server.orm.entity.Stage; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +@Slf4j +public class StageRunners { + + private static final AtomicBoolean LOADED = new AtomicBoolean(false); + + private static final Map STAGE_RUNNERS = new HashMap<>(); + + public static StageRunner getStageRunner(Stage stage) { + if (!LOADED.get()) { + load(); + } + + StageContext context = JsonUtils.readFromString(stage.getContext()); + StageType stageType = context.getStageType(); + + String beanName = STAGE_RUNNERS.get(stageType); + StageRunner runner = SpringContextHolder.getApplicationContext().getBean(beanName, StageRunner.class); + runner.setStage(stage); + + return runner; + } + + private static synchronized void load() { + if (LOADED.get()) { + return; + } + + for (Map.Entry entry : SpringContextHolder.getStageRunners().entrySet()) { + String beanName = entry.getKey(); + StageRunner stageRunner = entry.getValue(); + if (STAGE_RUNNERS.containsKey(stageRunner.getStageType())) { + log.error("Duplicate StageLifecycle with type: {}", stageRunner.getStageType()); + continue; + } + + STAGE_RUNNERS.put(stageRunner.getStageType(), beanName); + log.info("Load StageLifecycle: {} with type: {}", stageRunner.getClass().getName(), stageRunner.getStageType()); + } + + LOADED.set(true); + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/service/ServiceCustomJobFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/host/HostCacheStageRunner.java similarity index 34% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/service/ServiceCustomJobFactory.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/host/HostCacheStageRunner.java index 574d5d86..af65d48d 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/service/ServiceCustomJobFactory.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/host/HostCacheStageRunner.java @@ -1,20 +1,18 @@ -package org.apache.bigtop.manager.server.job.factory.service; +package org.apache.bigtop.manager.server.command.stage.runner.host; import lombok.extern.slf4j.Slf4j; -import org.apache.bigtop.manager.common.enums.Command; -import org.apache.bigtop.manager.server.enums.CommandLevel; -import org.apache.bigtop.manager.server.job.CommandIdentifier; -import org.apache.bigtop.manager.server.job.strategy.StageCallback; +import org.apache.bigtop.manager.server.command.stage.factory.StageType; +import org.apache.bigtop.manager.server.command.stage.runner.AbstractStageRunner; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Scope; @Slf4j @org.springframework.stereotype.Component @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) -public class ServiceCustomJobFactory extends AbstractServiceJobFactory implements StageCallback { +public class HostCacheStageRunner extends AbstractStageRunner { @Override - public CommandIdentifier getCommandIdentifier() { - return new CommandIdentifier(CommandLevel.SERVICE, Command.CUSTOM_COMMAND); + public StageType getStageType() { + return StageType.HOST_CACHE; } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/host/HostCheckStageRunner.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/host/HostCheckStageRunner.java new file mode 100644 index 00000000..fd11f78b --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/host/HostCheckStageRunner.java @@ -0,0 +1,17 @@ +package org.apache.bigtop.manager.server.command.stage.runner.host; + +import lombok.extern.slf4j.Slf4j; +import org.apache.bigtop.manager.server.command.stage.factory.StageType; +import org.apache.bigtop.manager.server.command.stage.runner.AbstractStageRunner; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; + +@Slf4j +@org.springframework.stereotype.Component +@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) +public class HostCheckStageRunner extends AbstractStageRunner { + + public StageType getStageType() { + return StageType.HOST_CHECK; + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/strategy/AbstractJobStrategy.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/strategy/AbstractJobStrategy.java similarity index 93% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/strategy/AbstractJobStrategy.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/strategy/AbstractJobStrategy.java index 99083828..2ea6f99c 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/strategy/AbstractJobStrategy.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/strategy/AbstractJobStrategy.java @@ -1,4 +1,4 @@ -package org.apache.bigtop.manager.server.job.strategy; +package org.apache.bigtop.manager.server.command.strategy; import jakarta.annotation.Resource; import org.apache.bigtop.manager.server.exception.ServerException; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/strategy/AsyncJobStrategy.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/strategy/AsyncJobStrategy.java similarity index 99% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/strategy/AsyncJobStrategy.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/strategy/AsyncJobStrategy.java index cffa5c75..789d2d78 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/strategy/AsyncJobStrategy.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/strategy/AsyncJobStrategy.java @@ -1,4 +1,4 @@ -package org.apache.bigtop.manager.server.job.strategy; +package org.apache.bigtop.manager.server.command.strategy; import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/strategy/JobStrategy.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/strategy/JobStrategy.java similarity index 78% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/strategy/JobStrategy.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/strategy/JobStrategy.java index 63bf2894..8807d726 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/strategy/JobStrategy.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/strategy/JobStrategy.java @@ -1,4 +1,4 @@ -package org.apache.bigtop.manager.server.job.strategy; +package org.apache.bigtop.manager.server.command.strategy; import org.apache.bigtop.manager.server.enums.JobStrategyType; import org.apache.bigtop.manager.server.orm.entity.Job; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/strategy/StageCallback.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/strategy/StageCallback.java similarity index 85% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/strategy/StageCallback.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/strategy/StageCallback.java index b6c8772b..95b17bcf 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/strategy/StageCallback.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/strategy/StageCallback.java @@ -1,4 +1,4 @@ -package org.apache.bigtop.manager.server.job.strategy; +package org.apache.bigtop.manager.server.command.strategy; import org.apache.bigtop.manager.server.orm.entity.Stage; import org.apache.bigtop.manager.server.orm.entity.Task; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/strategy/SyncJobStrategy.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/strategy/SyncJobStrategy.java similarity index 98% rename from bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/strategy/SyncJobStrategy.java rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/strategy/SyncJobStrategy.java index d2b71c5c..5686038a 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/strategy/SyncJobStrategy.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/strategy/SyncJobStrategy.java @@ -1,4 +1,4 @@ -package org.apache.bigtop.manager.server.job.strategy; +package org.apache.bigtop.manager.server.command.strategy; import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/holder/SpringContextHolder.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/holder/SpringContextHolder.java index 01b82fbc..f5c9980c 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/holder/SpringContextHolder.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/holder/SpringContextHolder.java @@ -18,9 +18,11 @@ package org.apache.bigtop.manager.server.holder; import jakarta.annotation.Nonnull; import lombok.Getter; -import org.apache.bigtop.manager.server.job.factory.JobFactory; -import org.apache.bigtop.manager.server.job.stage.StageLifecycle; -import org.apache.bigtop.manager.server.job.validator.CommandValidator; +import org.apache.bigtop.manager.server.command.job.factory.JobFactory; +import org.apache.bigtop.manager.server.command.job.runner.JobRunner; +import org.apache.bigtop.manager.server.command.stage.factory.StageFactory; +import org.apache.bigtop.manager.server.command.job.validator.CommandValidator; +import org.apache.bigtop.manager.server.command.stage.runner.StageRunner; import org.apache.bigtop.manager.server.ws.ServerWebSocketHandler; import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContextAware; @@ -51,7 +53,15 @@ public class SpringContextHolder implements ApplicationContextAware { return applicationContext.getBeansOfType(JobFactory.class); } - public static Map getStageLifecycles() { - return applicationContext.getBeansOfType(StageLifecycle.class); + public static Map getJobRunners() { + return applicationContext.getBeansOfType(JobRunner.class); + } + + public static Map getStageFactories() { + return applicationContext.getBeansOfType(StageFactory.class); + } + + public static Map getStageRunners() { + return applicationContext.getBeansOfType(StageRunner.class); } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/cluster/ClusterCreateJobFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/cluster/ClusterCreateJobFactory.java deleted file mode 100644 index 37582485..00000000 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/factory/cluster/ClusterCreateJobFactory.java +++ /dev/null @@ -1,144 +0,0 @@ -package org.apache.bigtop.manager.server.job.factory.cluster; - -import jakarta.annotation.Resource; -import lombok.extern.slf4j.Slf4j; -import org.apache.bigtop.manager.common.constants.Constants; -import org.apache.bigtop.manager.common.enums.Command; -import org.apache.bigtop.manager.common.message.type.RequestMessage; -import org.apache.bigtop.manager.common.message.type.pojo.ClusterInfo; -import org.apache.bigtop.manager.common.message.type.pojo.ComponentInfo; -import org.apache.bigtop.manager.common.message.type.pojo.RepoInfo; -import org.apache.bigtop.manager.common.utils.JsonUtils; -import org.apache.bigtop.manager.server.enums.CommandLevel; -import org.apache.bigtop.manager.server.job.CommandIdentifier; -import org.apache.bigtop.manager.server.job.helper.HostCacheStageHelper; -import org.apache.bigtop.manager.server.job.helper.HostCheckStageHelper; -import org.apache.bigtop.manager.server.job.strategy.StageCallback; -import org.apache.bigtop.manager.server.model.dto.ClusterDTO; -import org.apache.bigtop.manager.server.model.dto.ServiceDTO; -import org.apache.bigtop.manager.server.model.dto.StackDTO; -import org.apache.bigtop.manager.server.model.dto.command.ClusterCommandDTO; -import org.apache.bigtop.manager.server.model.mapper.RepoMapper; -import org.apache.bigtop.manager.server.orm.entity.Stage; -import org.apache.bigtop.manager.server.orm.entity.Task; -import org.apache.bigtop.manager.server.service.ClusterService; -import org.apache.bigtop.manager.server.utils.StackUtils; -import org.apache.commons.lang3.tuple.ImmutablePair; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.config.ConfigurableBeanFactory; -import org.springframework.context.annotation.Scope; - -import java.util.*; - -import static org.apache.bigtop.manager.common.constants.Constants.CACHE_STAGE_NAME; - -@Slf4j -@org.springframework.stereotype.Component -@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) -public class ClusterCreateJobFactory extends AbstractClusterJobFactory implements StageCallback { - - @Resource - private HostCheckStageHelper hostCheckStageHelper; - - @Resource - private HostCacheStageHelper hostCacheStageHelper; - - @Resource - private ClusterService clusterService; - - @Override - public CommandIdentifier getCommandIdentifier() { - return new CommandIdentifier(CommandLevel.CLUSTER, Command.CREATE); - } - - @Override - public List createStagesAndTasks() { - List stages = new ArrayList<>(); - String callbackClassName = this.getClass().getName(); - ClusterCommandDTO clusterCommand = jobContext.getCommandDTO().getClusterCommand(); - List hostnames = clusterCommand.getHostnames(); - - stages.add(hostCheckStageHelper.createStage(clusterCommand.getStackName(), clusterCommand.getStackVersion(), hostnames, callbackClassName)); - stages.add(createCacheStage(clusterCommand, callbackClassName, JsonUtils.writeAsString(clusterCommand))); - - return stages; - } - - @Override - public void beforeStage(Stage stage) { - if (stage.getName().equals(CACHE_STAGE_NAME)) { - ClusterCommandDTO clusterCommand = JsonUtils.readFromString(stage.getPayload(), ClusterCommandDTO.class); - ClusterDTO clusterDTO = new ClusterDTO(); - BeanUtils.copyProperties(clusterCommand, clusterDTO); - - clusterService.save(clusterDTO); - } - } - - public Stage createCacheStage(ClusterCommandDTO clusterCommand, String callbackClassName, String payload) { - Map componentInfoMap = new HashMap<>(); - Map> serviceConfigMap = new HashMap<>(); - Map> hostMap = new HashMap<>(); - Map> userMap = new HashMap<>(); - Map settingsMap = new HashMap<>(); - - String fullStackName = StackUtils.fullStackName(clusterCommand.getStackName(), clusterCommand.getStackVersion()); - ImmutablePair> immutablePair = StackUtils.getStackKeyMap().get(fullStackName); - StackDTO stackDTO = immutablePair.getLeft(); - List serviceDTOList = immutablePair.getRight(); - - List repoList = RepoMapper.INSTANCE.fromDTO2Message(clusterCommand.getRepoInfoList()); - ClusterInfo clusterInfo = new ClusterInfo(); - clusterInfo.setClusterName(clusterCommand.getClusterName()); - clusterInfo.setStackName(clusterCommand.getStackName()); - clusterInfo.setStackVersion(clusterCommand.getStackVersion()); - clusterInfo.setUserGroup(stackDTO.getUserGroup()); - clusterInfo.setRepoTemplate(stackDTO.getRepoTemplate()); - clusterInfo.setRoot(stackDTO.getRoot()); - - List hostnames = clusterCommand.getHostnames(); - hostMap.put(Constants.ALL_HOST_KEY, new HashSet<>(hostnames)); - - for (ServiceDTO serviceDTO : serviceDTOList) { - userMap.put(serviceDTO.getServiceUser(), Set.of(serviceDTO.getServiceGroup())); - } - - Stage hostCacheStage = new Stage(); - hostCacheStage.setName(CACHE_STAGE_NAME); - hostCacheStage.setCallbackClassName(callbackClassName); - hostCacheStage.setPayload(payload); - - List tasks = new ArrayList<>(); - for (String hostname : hostnames) { - Task task = new Task(); - task.setName("Cache host for " + hostname); - task.setStackName(clusterCommand.getStackName()); - task.setStackVersion(clusterCommand.getStackVersion()); - task.setHostname(hostname); - task.setServiceName("cluster"); - task.setServiceUser("root"); - task.setServiceGroup("root"); - task.setComponentName("bigtop-manager-agent"); - task.setCommand(Command.CUSTOM_COMMAND); - task.setCustomCommand("cache_host"); - - RequestMessage requestMessage = hostCacheStageHelper.getMessage( - hostname, - settingsMap, - clusterInfo, - serviceConfigMap, - hostMap, - repoList, - userMap, - componentInfoMap); - - log.info("[HostCacheJobFactory-requestMessage]: {}", requestMessage); - task.setContent(JsonUtils.writeAsString(requestMessage)); - task.setMessageId(requestMessage.getMessageId()); - tasks.add(task); - } - - hostCacheStage.setTasks(tasks); - return hostCacheStage; - } -} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/AbstractStageLifecycle.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/AbstractStageLifecycle.java deleted file mode 100644 index f601b214..00000000 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/AbstractStageLifecycle.java +++ /dev/null @@ -1,24 +0,0 @@ -package org.apache.bigtop.manager.server.job.stage; - -import lombok.Setter; - -@Setter -public abstract class AbstractStageLifecycle implements StageLifecycle { - - protected StageLifecycleContext context; - - @Override - public void beforeStage() { - - } - - @Override - public void onStageSuccess() { - - } - - @Override - public void onStageFailed() { - - } -} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/StageLifecycle.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/StageLifecycle.java deleted file mode 100644 index 598b4cd2..00000000 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/StageLifecycle.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.apache.bigtop.manager.server.job.stage; - -import org.apache.bigtop.manager.server.orm.entity.Stage; - -/** - * The StageLifecycle interface defines the lifecycle of a stage in a job. - * It provides methods for creating a stage, performing actions before the stage, - * handling successful completion of the stage, and handling failure of the stage. - */ -public interface StageLifecycle { - - /** - * Gets the type of the stage. - * - * @return the type of the stage - */ - StageType getStageType(); - - /** - * Creates a new Stage. - * - * @return a new Stage instance - */ - Stage createStage(); - - /** - * Performs actions before the stage starts. - */ - void beforeStage(); - - /** - * Handles the successful completion of the stage. - */ - void onStageSuccess(); - - /** - * Handles the failure of the stage. - */ - void onStageFailed(); -} \ No newline at end of file diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/StageLifecycleContext.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/StageLifecycleContext.java deleted file mode 100644 index 37722ef2..00000000 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/job/stage/StageLifecycleContext.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.apache.bigtop.manager.server.job.stage; - -import lombok.Data; -import org.apache.bigtop.manager.server.model.dto.RepoDTO; - -import java.util.List; - -@Data -public class StageLifecycleContext { - - private Long clusterId; - - private String clusterName; - - private String stackName; - - private String stackVersion; - - private List hostnames; - - private String serviceName; - - private String componentName; - - private List repoInfoList; -} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/orm/entity/Stage.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/orm/entity/Stage.java index d0954a23..dc5cac81 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/orm/entity/Stage.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/orm/entity/Stage.java @@ -42,6 +42,11 @@ public class Stage extends BaseEntity { @Column(name = "payload", length = 16777216) private String payload; + @Lob + @Basic(fetch = FetchType.LAZY) + @Column(name = "context", length = 16777216) + private String context; + @ManyToOne @JoinColumn(name = "job_id", foreignKey = @ForeignKey(ConstraintMode.NO_CONSTRAINT)) private Job job; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/CommandServiceImpl.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/CommandServiceImpl.java index 2aede94d..f8033378 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/CommandServiceImpl.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/CommandServiceImpl.java @@ -1,14 +1,14 @@ package org.apache.bigtop.manager.server.service.impl; +import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; -import org.apache.bigtop.manager.server.holder.SpringContextHolder; -import org.apache.bigtop.manager.server.job.CommandIdentifier; -import org.apache.bigtop.manager.server.job.event.CommandEvent; -import org.apache.bigtop.manager.server.job.factory.JobContext; -import org.apache.bigtop.manager.server.job.factory.JobFactories; -import org.apache.bigtop.manager.server.job.factory.JobFactory; -import org.apache.bigtop.manager.server.job.validator.ValidatorExecutionChain; -import org.apache.bigtop.manager.server.job.validator.ValidatorContext; +import org.apache.bigtop.manager.server.command.CommandIdentifier; +import org.apache.bigtop.manager.server.command.job.factory.JobContext; +import org.apache.bigtop.manager.server.command.job.factory.JobFactories; +import org.apache.bigtop.manager.server.command.job.factory.JobFactory; +import org.apache.bigtop.manager.server.command.job.scheduler.JobScheduler; +import org.apache.bigtop.manager.server.command.job.validator.ValidatorContext; +import org.apache.bigtop.manager.server.command.job.validator.ValidatorExecutionChain; import org.apache.bigtop.manager.server.model.dto.CommandDTO; import org.apache.bigtop.manager.server.model.mapper.JobMapper; import org.apache.bigtop.manager.server.model.vo.CommandVO; @@ -20,6 +20,9 @@ import org.springframework.transaction.annotation.Transactional; @org.springframework.stereotype.Service public class CommandServiceImpl implements CommandService { + @Resource + private JobScheduler jobScheduler; + @Override @Transactional public CommandVO command(CommandDTO commandDTO) { @@ -36,10 +39,13 @@ public class CommandServiceImpl implements CommandService { JobFactory jobFactory = JobFactories.getJobFactory(commandIdentifier); Job job = jobFactory.createJob(jobContext); + // Submit job to scheduler + jobScheduler.submit(job); + // Publish command event - CommandEvent event = new CommandEvent(commandDTO); - event.setJobId(job.getId()); - SpringContextHolder.getApplicationContext().publishEvent(event); +// CommandEvent event = new CommandEvent(commandDTO); +// event.setJobId(job.getId()); +// SpringContextHolder.getApplicationContext().publishEvent(event); return JobMapper.INSTANCE.fromEntity2CommandVO(job); } -- Gitee From 745078c060a69b2e8fec6a632c24c9f46d36b4aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E6=B2=BB=E5=9B=BD?= Date: Sat, 3 Feb 2024 17:56:50 +0800 Subject: [PATCH 3/4] 1 --- README.md | 1 - .../common/message/type/CommandPayload.java | 5 +- .../job/factory/AbstractJobFactory.java | 3 +- .../AbstractComponentJobFactory.java | 189 ------------ .../component/ComponentInstallJobFactory.java | 40 +-- .../component/ComponentRestartJobFactory.java | 10 +- .../component/ComponentStartJobFactory.java | 50 +--- .../component/ComponentStopJobFactory.java | 48 +--- .../job/factory/host/HostAddJobFactory.java | 112 +------- .../service/AbstractServiceJobFactory.java | 202 +++++++++++-- .../service/ServiceConfigureJobFactory.java | 39 +-- .../service/ServiceInstallJobFactory.java | 157 ++-------- .../service/ServiceRestartJobFactory.java | 14 +- .../service/ServiceStartJobFactory.java | 40 +-- .../service/ServiceStopJobFactory.java | 40 +-- .../job/helper/HostCacheStageHelper.java | 269 ------------------ .../job/helper/HostCheckStageHelper.java | 86 ------ .../command/job/runner/AbstractJobRunner.java | 6 + .../cluster/ClusterCreateJobRunner.java | 42 ++- .../component/ComponentInstallJobRunner.java | 20 ++ .../component/ComponentRestartJobRunner.java | 20 ++ .../component/ComponentStartJobRunner.java | 20 ++ .../component/ComponentStopJobRunner.java | 20 ++ .../job/runner/host/HostAddJobRunner.java | 41 +++ .../service/ServiceConfigureJobRunner.java | 20 ++ .../service/ServiceInstallJobRunner.java | 32 +++ .../service/ServiceRestartJobRunner.java | 20 ++ .../runner/service/ServiceStartJobRunner.java | 20 ++ .../runner/service/ServiceStopJobRunner.java | 20 ++ .../listener/CommandEventListener.java | 54 ---- .../command/stage/factory/StageContext.java | 8 +- .../command/stage/factory/StageType.java | 2 +- .../AbstractComponentStageFactory.java | 141 +++++++++ .../component/ComponentCheckStageFactory.java | 23 ++ .../ComponentInstallStageFactory.java | 7 +- .../component/ComponentStartStageFactory.java | 23 ++ .../component/ComponentStopStageFactory.java | 23 ++ .../factory/host/HostCacheStageFactory.java | 10 +- .../service/ServiceConfigureStageFactory.java | 22 ++ .../stage/runner/AbstractStageRunner.java | 196 ++++++++++++- .../command/stage/runner/StageRunner.java | 7 + .../command/stage/runner/StageRunners.java | 2 +- .../component/ComponentCheckStageRunner.java | 20 ++ .../ComponentInstallStageRunner.java | 38 +++ .../component/ComponentStartStageRunner.java | 43 +++ .../component/ComponentStopStageRunner.java | 39 +++ .../service/ServiceConfigureStageRunner.java | 18 ++ .../command/strategy/AbstractJobStrategy.java | 26 -- .../command/strategy/AsyncJobStrategy.java | 184 ------------ .../server/command/strategy/JobStrategy.java | 8 - .../command/strategy/StageCallback.java | 17 -- .../command/strategy/SyncJobStrategy.java | 109 ------- .../server/model/mapper/ClusterMapper.java | 3 + .../manager/server/orm/entity/Service.java | 7 - .../manager/server/orm/entity/Stage.java | 9 - .../repository/HostComponentRepository.java | 2 + .../service/impl/CommandServiceImpl.java | 16 +- .../service/impl/ServiceServiceImpl.java | 6 +- .../stack/common/utils/LocalSettings.java | 4 +- .../stack/core/hooks/InstallHookImpl.java | 2 +- 60 files changed, 1151 insertions(+), 1504 deletions(-) delete mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/helper/HostCacheStageHelper.java delete mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/helper/HostCheckStageHelper.java create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/component/ComponentInstallJobRunner.java create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/component/ComponentRestartJobRunner.java create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/component/ComponentStartJobRunner.java create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/component/ComponentStopJobRunner.java create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/host/HostAddJobRunner.java create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/service/ServiceConfigureJobRunner.java create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/service/ServiceInstallJobRunner.java create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/service/ServiceRestartJobRunner.java create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/service/ServiceStartJobRunner.java create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/service/ServiceStopJobRunner.java create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/component/AbstractComponentStageFactory.java create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/component/ComponentCheckStageFactory.java create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/component/ComponentStartStageFactory.java create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/component/ComponentStopStageFactory.java create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/service/ServiceConfigureStageFactory.java create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/component/ComponentCheckStageRunner.java create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/component/ComponentInstallStageRunner.java create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/component/ComponentStartStageRunner.java create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/component/ComponentStopStageRunner.java create mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/service/ServiceConfigureStageRunner.java delete mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/strategy/AbstractJobStrategy.java delete mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/strategy/AsyncJobStrategy.java delete mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/strategy/JobStrategy.java delete mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/strategy/StageCallback.java delete mode 100644 bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/strategy/SyncJobStrategy.java diff --git a/README.md b/README.md index f8114010..3e65bd37 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,5 @@ # Bigtop-Manager -Bigtop Manager 是一款现代化的，面向云原生的大数据集群管理平台，提供了一套轻量的方案来高效地管理 Bigtop 所支持的大数据组件 Bigtop-Manager is a platform for managing Bigtop components. Inspired by Apache Ambari. ## Prerequisites diff --git a/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/message/type/CommandPayload.java b/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/message/type/CommandPayload.java index aaa647f4..a73e50d0 100644 --- a/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/message/type/CommandPayload.java +++ b/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/message/type/CommandPayload.java @@ -1,13 +1,14 @@ package org.apache.bigtop.manager.common.message.type; -import lombok.*; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.ToString; import org.apache.bigtop.manager.common.enums.Command; import org.apache.bigtop.manager.common.message.type.pojo.CustomCommandInfo; import org.apache.bigtop.manager.common.message.type.pojo.OSSpecificInfo; import org.apache.bigtop.manager.common.message.type.pojo.ScriptInfo; import java.util.List; -import java.util.Map; @Data @ToString(callSuper = true) diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/AbstractJobFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/AbstractJobFactory.java index 8e389bb9..cce2e0d2 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/AbstractJobFactory.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/AbstractJobFactory.java @@ -45,7 +45,7 @@ public abstract class AbstractJobFactory implements JobFactory { initJob(); // Create stages and tasks for job - job.setStages(stages); + createStagesAndTasks(); // Save job saveJob(); @@ -64,6 +64,7 @@ public abstract class AbstractJobFactory implements JobFactory { job.setState(JobState.PENDING); job.setCluster(cluster); job.setPayload(JsonUtils.writeAsString(jobContext.getCommandDTO())); + job.setStages(stages); } protected void saveJob() { diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/component/AbstractComponentJobFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/component/AbstractComponentJobFactory.java index 8d18f924..06664f28 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/component/AbstractComponentJobFactory.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/component/AbstractComponentJobFactory.java @@ -1,196 +1,7 @@ package org.apache.bigtop.manager.server.command.job.factory.component; -import com.fasterxml.jackson.core.type.TypeReference; -import jakarta.annotation.Resource; -import org.apache.bigtop.manager.common.enums.Command; -import org.apache.bigtop.manager.common.enums.MessageType; -import org.apache.bigtop.manager.common.message.type.CommandPayload; -import org.apache.bigtop.manager.common.message.type.RequestMessage; -import org.apache.bigtop.manager.common.message.type.pojo.ScriptInfo; -import org.apache.bigtop.manager.common.utils.JsonUtils; import org.apache.bigtop.manager.server.command.job.factory.AbstractJobFactory; -import org.apache.bigtop.manager.server.enums.ApiExceptionEnum; -import org.apache.bigtop.manager.server.exception.ApiException; -import org.apache.bigtop.manager.server.exception.ServerException; -import org.apache.bigtop.manager.server.model.dto.CommandDTO; -import org.apache.bigtop.manager.server.model.dto.command.ComponentCommandDTO; -import org.apache.bigtop.manager.server.orm.entity.Component; -import org.apache.bigtop.manager.server.orm.entity.Stage; -import org.apache.bigtop.manager.server.orm.entity.Task; -import org.apache.bigtop.manager.server.orm.repository.ComponentRepository; -import org.apache.bigtop.manager.server.stack.dag.ComponentCommandWrapper; -import org.apache.bigtop.manager.server.stack.dag.DAG; -import org.apache.bigtop.manager.server.stack.dag.DagGraphEdge; -import org.apache.bigtop.manager.server.utils.StackUtils; -import org.apache.commons.text.CaseUtils; -import org.springframework.beans.BeanUtils; - -import java.text.MessageFormat; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; public abstract class AbstractComponentJobFactory extends AbstractJobFactory { - @Resource - private ComponentRepository componentRepository; - - @Override - protected void createStagesAndTasks() { - createStages(getClass().getName()); - } - - protected void createStages(String callbackClassName) { - createStages(callbackClassName, jobContext.getCommandDTO().getCommand()); - } - - protected void createStages(String callbackClassName, Command command) { - CommandDTO commandDTO = jobContext.getCommandDTO(); - String customCommand = commandDTO.getCustomCommand(); - - List sortedCommandWrappers = getSortedCommandWrappers(command); - Map> componentHostMapping = getComponentHostMapping(command); - - for (ComponentCommandWrapper componentCommandWrapper : sortedCommandWrappers) { - String componentName = componentCommandWrapper.getComponentName(); - Component component = componentCommandWrapper.getComponent(); - if (component == null) { - throw new ApiException(ApiExceptionEnum.COMPONENT_NOT_FOUND); - } - - Stage stage = createStage(callbackClassName, componentCommandWrapper, component, command); - stages.add(stage); - - // Generate task list - List tasks = new ArrayList<>(); - List hostnames = componentHostMapping.get(componentName); - for (String hostname : hostnames) { - Task task = createTask(component, hostname, command, customCommand); - tasks.add(task); - } - - stage.setTasks(tasks); - } - } - - protected Map> getComponentHostMapping(Command command) { - List componentCommands = jobContext.getCommandDTO().getComponentCommands(); - return componentCommands.stream().collect(Collectors.toMap(ComponentCommandDTO::getComponentName, ComponentCommandDTO::getHostnames)); - } - - protected List getComponents() { - List componentCommands = jobContext.getCommandDTO().getComponentCommands(); - List componentNames = componentCommands.stream().map(ComponentCommandDTO::getComponentName).toList(); - - return componentRepository.findAllByClusterIdAndComponentNameIn(cluster.getId(), componentNames); - } - - private List getSortedCommandWrappers(Command command) { - String stackName = cluster.getStack().getStackName(); - String stackVersion = cluster.getStack().getStackVersion(); - String fullStackName = StackUtils.fullStackName(stackName, stackVersion); - - DAG dag = StackUtils.getStackDagMap().get(fullStackName); - try { - Map commandWrapperMap = getCommandWrappers(command).stream() - .collect(Collectors.toMap(ComponentCommandWrapper::toString, Function.identity())); - - List orderedList = dag.topologicalSort(); - List todoList = new ArrayList<>(commandWrapperMap.keySet()); - - orderedList.retainAll(todoList); - todoList.removeAll(orderedList); - orderedList.addAll(todoList); - - return orderedList.stream().map(commandWrapperMap::get).collect(Collectors.toList()); - } catch (Exception e) { - throw new ServerException(e); - } - } - - private List getCommandWrappers(Command command) { - return getComponents().stream() - .map(component -> new ComponentCommandWrapper(component.getComponentName(), command, component)) - .collect(Collectors.toList()); - } - - private Stage createStage(String callbackClassName, ComponentCommandWrapper componentCommandWrapper, Component component, Command command) { - Stage stage = new Stage(); - stage.setName(componentCommandWrapper.toDisplayString()); - stage.setServiceName(component.getService().getServiceName()); - stage.setComponentName(component.getComponentName()); - - // Set stage callback - stage.setCallbackClassName(callbackClassName); - - if (command == jobContext.getCommandDTO().getCommand()) { - stage.setPayload(JsonUtils.writeAsString(jobContext.getCommandDTO())); - } else { - CommandDTO commandDTO = new CommandDTO(); - BeanUtils.copyProperties(jobContext.getCommandDTO(), commandDTO); - commandDTO.setCommand(command); - stage.setPayload(JsonUtils.writeAsString(commandDTO)); - } - - return stage; - } - - private Task createTask(Component component, String hostname, Command command, String customCommand) { - Task task = new Task(); - - task.setName(MessageFormat.format("{0} for {1} and {2}", - CaseUtils.toCamelCase(command.name(), true), component.getDisplayName(), hostname)); - - // Required fields - task.setHostname(hostname); - task.setCommand(command); - task.setServiceName(component.getService().getServiceName()); - task.setStackName(component.getCluster().getStack().getStackName()); - task.setStackVersion(component.getCluster().getStack().getStackVersion()); - - // Context fields - task.setComponentName(component.getComponentName()); - task.setServiceUser(component.getService().getServiceUser()); - task.setServiceGroup(component.getService().getServiceGroup()); - task.setCustomCommands(component.getCustomCommands()); - task.setCustomCommand(customCommand); - task.setCommandScript(component.getCommandScript()); - - RequestMessage requestMessage = getMessage(component, hostname, command, customCommand); - task.setContent(JsonUtils.writeAsString(requestMessage)); - task.setMessageId(requestMessage.getMessageId()); - - return task; - } - - private RequestMessage getMessage(Component component, String hostname, Command command, String customCommand) { - RequestMessage requestMessage = new RequestMessage(); - requestMessage.setMessageType(MessageType.COMMAND); - requestMessage.setHostname(hostname); - requestMessage.setMessagePayload(JsonUtils.writeAsString(getMessagePayload(component, hostname, command, customCommand))); - - return requestMessage; - } - - private CommandPayload getMessagePayload(Component component, String hostname, Command command, String customCommand) { - CommandPayload commandPayload = new CommandPayload(); - commandPayload.setServiceName(component.getService().getServiceName()); - commandPayload.setCommand(command); - commandPayload.setCustomCommand(customCommand); - commandPayload.setServiceUser(component.getService().getServiceUser()); - commandPayload.setServiceGroup(component.getService().getServiceGroup()); - commandPayload.setStackName(component.getCluster().getStack().getStackName()); - commandPayload.setStackVersion(component.getCluster().getStack().getStackVersion()); - commandPayload.setRoot(component.getService().getCluster().getRoot()); - commandPayload.setComponentName(component.getComponentName()); - commandPayload.setHostname(hostname); - - commandPayload.setCustomCommands(JsonUtils.readFromString(component.getCustomCommands(), new TypeReference<>() {})); - commandPayload.setOsSpecifics(JsonUtils.readFromString(component.getService().getOsSpecifics(), new TypeReference<>() {})); - commandPayload.setCommandScript(JsonUtils.readFromString(component.getCommandScript(), ScriptInfo.class)); - - return commandPayload; - } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/component/ComponentInstallJobFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/component/ComponentInstallJobFactory.java index 1b0528b0..5123fe8c 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/component/ComponentInstallJobFactory.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/component/ComponentInstallJobFactory.java @@ -1,34 +1,16 @@ package org.apache.bigtop.manager.server.command.job.factory.component; -import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.apache.bigtop.manager.common.enums.Command; -import org.apache.bigtop.manager.common.message.type.RequestMessage; -import org.apache.bigtop.manager.common.utils.JsonUtils; import org.apache.bigtop.manager.server.command.CommandIdentifier; -import org.apache.bigtop.manager.server.command.job.helper.HostCacheStageHelper; -import org.apache.bigtop.manager.server.command.strategy.StageCallback; import org.apache.bigtop.manager.server.enums.CommandLevel; -import org.apache.bigtop.manager.server.model.dto.CommandDTO; -import org.apache.bigtop.manager.server.orm.entity.Cluster; -import org.apache.bigtop.manager.server.orm.entity.Stage; -import org.apache.bigtop.manager.server.orm.entity.Task; -import org.apache.bigtop.manager.server.service.HostComponentService; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Scope; -import static org.apache.bigtop.manager.common.constants.Constants.CACHE_STAGE_NAME; - @Slf4j @org.springframework.stereotype.Component @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) -public class ComponentInstallJobFactory extends AbstractComponentJobFactory implements StageCallback { - - @Resource - private HostCacheStageHelper hostCacheStageHelper; - - @Resource - private HostComponentService hostComponentService; +public class ComponentInstallJobFactory extends AbstractComponentJobFactory { @Override public CommandIdentifier getCommandIdentifier() { @@ -37,26 +19,6 @@ public class ComponentInstallJobFactory extends AbstractComponentJobFactory impl @Override public void createStagesAndTasks() { - String callbackClassName = this.getClass().getName(); - String payload = JsonUtils.writeAsString(jobContext.getCommandDTO()); - stages.add(hostCacheStageHelper.createStage(cluster.getId(), callbackClassName, payload)); - } - @Override - public void beforeStage(Stage stage) { - CommandDTO commandDTO = JsonUtils.readFromString(stage.getPayload(), CommandDTO.class); - if (stage.getName().equals(CACHE_STAGE_NAME) && commandDTO.getCommand() == Command.INSTALL && commandDTO.getCommandLevel() == CommandLevel.COMPONENT) { - hostComponentService.saveByCommand(commandDTO); - } } - - @Override - public String generatePayload(Task task) { - Cluster cluster = task.getCluster(); - hostCacheStageHelper.createCache(cluster); - RequestMessage requestMessage = hostCacheStageHelper.getMessage(task.getHostname()); - log.info("[generatePayload]-[HostCacheJobFactory-requestMessage]: {}", requestMessage); - return JsonUtils.writeAsString(requestMessage); - } - } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/component/ComponentRestartJobFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/component/ComponentRestartJobFactory.java index b3a65ff4..7f35745a 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/component/ComponentRestartJobFactory.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/component/ComponentRestartJobFactory.java @@ -2,19 +2,23 @@ package org.apache.bigtop.manager.server.command.job.factory.component; import lombok.extern.slf4j.Slf4j; import org.apache.bigtop.manager.common.enums.Command; -import org.apache.bigtop.manager.server.enums.CommandLevel; import org.apache.bigtop.manager.server.command.CommandIdentifier; -import org.apache.bigtop.manager.server.command.strategy.StageCallback; +import org.apache.bigtop.manager.server.enums.CommandLevel; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Scope; @Slf4j @org.springframework.stereotype.Component @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) -public class ComponentRestartJobFactory extends AbstractComponentJobFactory implements StageCallback { +public class ComponentRestartJobFactory extends AbstractComponentJobFactory { @Override public CommandIdentifier getCommandIdentifier() { return new CommandIdentifier(CommandLevel.COMPONENT, Command.RESTART); } + + @Override + public void createStagesAndTasks() { + + } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/component/ComponentStartJobFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/component/ComponentStartJobFactory.java index 5815d121..ad3a46a8 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/component/ComponentStartJobFactory.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/component/ComponentStartJobFactory.java @@ -1,36 +1,16 @@ package org.apache.bigtop.manager.server.command.job.factory.component; -import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.apache.bigtop.manager.common.enums.Command; -import org.apache.bigtop.manager.common.utils.JsonUtils; -import org.apache.bigtop.manager.server.enums.CommandLevel; -import org.apache.bigtop.manager.server.enums.JobState; -import org.apache.bigtop.manager.server.enums.MaintainState; import org.apache.bigtop.manager.server.command.CommandIdentifier; -import org.apache.bigtop.manager.server.command.strategy.StageCallback; -import org.apache.bigtop.manager.server.model.dto.CommandDTO; -import org.apache.bigtop.manager.server.model.dto.command.ComponentCommandDTO; -import org.apache.bigtop.manager.server.orm.entity.HostComponent; -import org.apache.bigtop.manager.server.orm.entity.Service; -import org.apache.bigtop.manager.server.orm.entity.Stage; -import org.apache.bigtop.manager.server.orm.repository.HostComponentRepository; -import org.apache.bigtop.manager.server.orm.repository.ServiceRepository; +import org.apache.bigtop.manager.server.enums.CommandLevel; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Scope; -import java.util.List; - @Slf4j @org.springframework.stereotype.Component @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) -public class ComponentStartJobFactory extends AbstractComponentJobFactory implements StageCallback { - - @Resource - private ServiceRepository serviceRepository; - - @Resource - private HostComponentRepository hostComponentRepository; +public class ComponentStartJobFactory extends AbstractComponentJobFactory { @Override public CommandIdentifier getCommandIdentifier() { @@ -38,31 +18,7 @@ public class ComponentStartJobFactory extends AbstractComponentJobFactory implem } @Override - public void afterStage(Stage stage) { - Long clusterId = stage.getCluster().getId(); - String componentName = stage.getComponentName(); - CommandDTO commandDTO = JsonUtils.readFromString(stage.getPayload(), CommandDTO.class); - List componentCommands = commandDTO.getComponentCommands(); - List hostnames = componentCommands - .stream() - .filter(x -> x.getComponentName().equals(componentName)) - .flatMap(x -> x.getHostnames().stream()) - .toList(); - - if (stage.getState() == JobState.SUCCESSFUL) { - List hostComponents = hostComponentRepository - .findAllByComponentClusterIdAndComponentComponentNameAndHostHostnameIn(clusterId, componentName, hostnames); - Service service = hostComponents.get(0).getComponent().getService(); + public void createStagesAndTasks() { - // Update the state of the host component - hostComponents.forEach(hostComponent -> hostComponent.setState(MaintainState.STARTED)); - hostComponentRepository.saveAll(hostComponents); - - if (hostComponents.stream().allMatch(x -> x.getState() == MaintainState.STARTED)) { - service.setState(MaintainState.STARTED); - } - serviceRepository.save(service); - } } - } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/component/ComponentStopJobFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/component/ComponentStopJobFactory.java index 5e6363ac..7f381321 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/component/ComponentStopJobFactory.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/component/ComponentStopJobFactory.java @@ -1,34 +1,16 @@ package org.apache.bigtop.manager.server.command.job.factory.component; -import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.apache.bigtop.manager.common.enums.Command; -import org.apache.bigtop.manager.common.utils.JsonUtils; -import org.apache.bigtop.manager.server.enums.CommandLevel; -import org.apache.bigtop.manager.server.enums.JobState; -import org.apache.bigtop.manager.server.enums.MaintainState; import org.apache.bigtop.manager.server.command.CommandIdentifier; -import org.apache.bigtop.manager.server.command.strategy.StageCallback; -import org.apache.bigtop.manager.server.model.dto.CommandDTO; -import org.apache.bigtop.manager.server.model.dto.command.ComponentCommandDTO; -import org.apache.bigtop.manager.server.orm.entity.*; -import org.apache.bigtop.manager.server.orm.repository.HostComponentRepository; -import org.apache.bigtop.manager.server.orm.repository.ServiceRepository; +import org.apache.bigtop.manager.server.enums.CommandLevel; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Scope; -import java.util.List; - @Slf4j @org.springframework.stereotype.Component @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) -public class ComponentStopJobFactory extends AbstractComponentJobFactory implements StageCallback { - - @Resource - private ServiceRepository serviceRepository; - - @Resource - private HostComponentRepository hostComponentRepository; +public class ComponentStopJobFactory extends AbstractComponentJobFactory { @Override public CommandIdentifier getCommandIdentifier() { @@ -36,31 +18,7 @@ public class ComponentStopJobFactory extends AbstractComponentJobFactory impleme } @Override - public void afterStage(Stage stage) { - Long clusterId = stage.getCluster().getId(); - String componentName = stage.getComponentName(); - CommandDTO commandDTO = JsonUtils.readFromString(stage.getPayload(), CommandDTO.class); - List componentCommands = commandDTO.getComponentCommands(); - List hostnames = componentCommands - .stream() - .filter(x -> x.getComponentName().equals(componentName)) - .flatMap(x -> x.getHostnames().stream()) - .toList(); - - if (stage.getState() == JobState.SUCCESSFUL) { - List hostComponents = hostComponentRepository - .findAllByComponentClusterIdAndComponentComponentNameAndHostHostnameIn(clusterId, componentName, hostnames); - Service service = hostComponents.get(0).getComponent().getService(); + public void createStagesAndTasks() { - // Update the state of the host component - hostComponents.forEach(hostComponent -> hostComponent.setState(MaintainState.STOPPED)); - hostComponentRepository.saveAll(hostComponents); - - if (hostComponents.stream().allMatch(x -> x.getState() == MaintainState.STOPPED)) { - service.setState(MaintainState.STOPPED); - } - serviceRepository.save(service); - } } - } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/host/HostAddJobFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/host/HostAddJobFactory.java index 34a21a6f..a013b101 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/host/HostAddJobFactory.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/host/HostAddJobFactory.java @@ -1,125 +1,29 @@ package org.apache.bigtop.manager.server.command.job.factory.host; -import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.apache.bigtop.manager.common.enums.Command; -import org.apache.bigtop.manager.common.message.type.RequestMessage; import org.apache.bigtop.manager.common.utils.JsonUtils; -import org.apache.bigtop.manager.server.enums.CommandLevel; import org.apache.bigtop.manager.server.command.CommandIdentifier; -import org.apache.bigtop.manager.server.command.job.helper.HostCacheStageHelper; -import org.apache.bigtop.manager.server.command.job.helper.HostCheckStageHelper; -import org.apache.bigtop.manager.server.command.strategy.StageCallback; -import org.apache.bigtop.manager.server.model.dto.command.HostCommandDTO; -import org.apache.bigtop.manager.server.orm.entity.Cluster; -import org.apache.bigtop.manager.server.orm.entity.Host; -import org.apache.bigtop.manager.server.orm.entity.Stage; -import org.apache.bigtop.manager.server.orm.entity.Task; -import org.apache.bigtop.manager.server.orm.repository.HostRepository; -import org.apache.bigtop.manager.server.service.HostService; -import org.apache.commons.lang3.StringUtils; +import org.apache.bigtop.manager.server.command.stage.factory.StageContext; +import org.apache.bigtop.manager.server.command.stage.factory.StageFactories; +import org.apache.bigtop.manager.server.command.stage.factory.StageType; +import org.apache.bigtop.manager.server.enums.CommandLevel; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Scope; -import java.util.ArrayList; -import java.util.List; - -import static org.apache.bigtop.manager.common.constants.Constants.CACHE_STAGE_NAME; - @Slf4j @org.springframework.stereotype.Component @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) -public class HostAddJobFactory extends AbstractHostJobFactory implements StageCallback { - - @Resource - private HostRepository hostRepository; - - @Resource - private HostCacheStageHelper hostCacheStageHelper; - - @Resource - private HostService hostService; - - @Resource - private HostCheckStageHelper hostCheckStageHelper; +public class HostAddJobFactory extends AbstractHostJobFactory { @Override public CommandIdentifier getCommandIdentifier() { return new CommandIdentifier(CommandLevel.HOST, Command.INSTALL); } - @Override public void createStagesAndTasks() { - String callbackClassName = this.getClass().getName(); - Long clusterId = jobContext.getCommandDTO().getClusterId(); - List hostnames = jobContext.getCommandDTO().getHostCommands().stream().map(HostCommandDTO::getHostname).toList(); - - // Create stages - stages.add(hostCheckStageHelper.createStage(clusterId, hostnames, callbackClassName)); - stages.add(createStage(callbackClassName, JsonUtils.writeAsString(hostnames))); - } - - public Stage createStage(String callbackClassName, String payload) { - hostCacheStageHelper.createCache(cluster); - - List hostList = hostRepository.findAllByClusterId(cluster.getId()); - List hostnames = new ArrayList<>(hostList.stream().map(Host::getHostname).toList()); - hostnames.addAll(JsonUtils.readFromString(payload)); - - Stage hostCacheStage = new Stage(); - hostCacheStage.setName(CACHE_STAGE_NAME); - - if (StringUtils.isNotEmpty(callbackClassName)) { - hostCacheStage.setCallbackClassName(callbackClassName); - } else { - hostCacheStage.setCallbackClassName(this.getClass().getName()); - } - - if (StringUtils.isNotEmpty(callbackClassName)) { - hostCacheStage.setPayload(payload); - } - - List tasks = new ArrayList<>(); - for (String hostname : hostnames) { - Task task = new Task(); - task.setName("Cache host for " + hostname); - task.setStackName(cluster.getStack().getStackName()); - task.setStackVersion(cluster.getStack().getStackVersion()); - task.setHostname(hostname); - task.setServiceName("cluster"); - task.setServiceUser("root"); - task.setServiceGroup("root"); - task.setComponentName("bigtop-manager-agent"); - task.setCommand(Command.CUSTOM_COMMAND); - task.setCustomCommand("cache_host"); - - RequestMessage requestMessage = hostCheckStageHelper.createMessage(hostname); - log.info("[HostCacheJobFactory-requestMessage]: {}", requestMessage); - task.setContent(JsonUtils.writeAsString(requestMessage)); - task.setMessageId(requestMessage.getMessageId()); - - tasks.add(task); - } - - hostCacheStage.setTasks(tasks); - return hostCacheStage; - } - - @Override - public void beforeStage(Stage stage) { - Cluster cluster = stage.getCluster(); - if (stage.getName().equals(CACHE_STAGE_NAME)) { - List hostnames = JsonUtils.readFromString(stage.getPayload()); - hostService.batchSave(cluster.getId(), hostnames); - } - } - - @Override - public String generatePayload(Task task) { - Cluster cluster = task.getCluster(); - hostCacheStageHelper.createCache(cluster); - RequestMessage requestMessage = hostCacheStageHelper.getMessage(task.getHostname()); - log.info("[generatePayload]-[HostCacheJobFactory-requestMessage]: {}", requestMessage); - return JsonUtils.writeAsString(requestMessage); + StageContext stageContext = StageContext.fromPayload(JsonUtils.writeAsString(jobContext.getCommandDTO())); + stages.add(StageFactories.getStageFactory(StageType.HOST_CHECK).createStage(stageContext)); + stages.add(StageFactories.getStageFactory(StageType.HOST_CACHE).createStage(stageContext)); } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/AbstractServiceJobFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/AbstractServiceJobFactory.java index c7a0d380..37f743d9 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/AbstractServiceJobFactory.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/AbstractServiceJobFactory.java @@ -2,22 +2,38 @@ package org.apache.bigtop.manager.server.command.job.factory.service; import jakarta.annotation.Resource; import org.apache.bigtop.manager.common.enums.Command; -import org.apache.bigtop.manager.server.command.job.factory.component.AbstractComponentJobFactory; +import org.apache.bigtop.manager.common.utils.JsonUtils; +import org.apache.bigtop.manager.server.command.job.factory.AbstractJobFactory; +import org.apache.bigtop.manager.server.command.stage.factory.StageContext; +import org.apache.bigtop.manager.server.command.stage.factory.StageFactories; +import org.apache.bigtop.manager.server.command.stage.factory.StageType; +import org.apache.bigtop.manager.server.exception.ServerException; +import org.apache.bigtop.manager.server.model.dto.ComponentDTO; +import org.apache.bigtop.manager.server.model.dto.ServiceDTO; +import org.apache.bigtop.manager.server.model.dto.StackDTO; import org.apache.bigtop.manager.server.model.dto.command.ServiceCommandDTO; import org.apache.bigtop.manager.server.orm.entity.Component; +import org.apache.bigtop.manager.server.orm.entity.Host; import org.apache.bigtop.manager.server.orm.entity.HostComponent; import org.apache.bigtop.manager.server.orm.repository.ComponentRepository; import org.apache.bigtop.manager.server.orm.repository.HostComponentRepository; +import org.apache.bigtop.manager.server.stack.dag.ComponentCommandWrapper; +import org.apache.bigtop.manager.server.stack.dag.DAG; +import org.apache.bigtop.manager.server.stack.dag.DagGraphEdge; +import org.apache.bigtop.manager.server.utils.StackUtils; +import org.apache.commons.lang3.tuple.ImmutablePair; -import java.util.HashMap; +import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; /** * A Service Job can be seen as a collection of multiple Components and Hosts, * so it can directly inherit from AbstractComponentJobFactory. */ -public abstract class AbstractServiceJobFactory extends AbstractComponentJobFactory { +public abstract class AbstractServiceJobFactory extends AbstractJobFactory { @Resource private ComponentRepository componentRepository; @@ -25,27 +41,175 @@ public abstract class AbstractServiceJobFactory extends AbstractComponentJobFact @Resource private HostComponentRepository hostComponentRepository; - protected Map> getComponentHostMapping(Command command) { - Map> componentHostMapping = new HashMap<>(); - for (Component component : getComponents()) { - String componentName = component.getComponentName(); - List hostComponentList = hostComponentRepository - .findAllByComponentClusterIdAndComponentComponentName(cluster.getId(), componentName); + protected String stackName; - List hostnames = hostComponentList.stream().map(x -> x.getHost().getHostname()).toList(); - componentHostMapping.put(componentName, hostnames); + protected String stackVersion; + + protected Map serviceNameToDTO; + + protected Map componentNameToDTO; + + protected DAG dag; + + protected void initAttrs() { + stackName = cluster.getStack().getStackName(); + stackVersion = cluster.getStack().getStackVersion(); + + Map>> stackKeyMap = StackUtils.getStackKeyMap(); + ImmutablePair> immutablePair = stackKeyMap.get(StackUtils.fullStackName(stackName, stackVersion)); + + serviceNameToDTO = immutablePair.getRight() + .stream() + .collect(Collectors.toMap(ServiceDTO::getServiceName, Function.identity())); + + componentNameToDTO = immutablePair.getRight() + .stream() + .flatMap(serviceDTO -> serviceDTO.getComponents().stream()) + .collect(Collectors.toMap(ComponentDTO::getComponentName, Function.identity())); + + dag = StackUtils.getStackDagMap().get(StackUtils.fullStackName(stackName, stackVersion)); + } + + protected StageContext createStageContext(String serviceName, String componentName) { + return createStageContext(serviceName, componentName, findHostnamesByComponentName(componentName)); + } + + protected StageContext createStageContext(String serviceName, String componentName, List hostnames) { + ServiceDTO serviceDTO = serviceNameToDTO.get(serviceName); + ComponentDTO componentDTO = componentNameToDTO.get(componentName); + + StageContext stageContext = StageContext.fromPayload(JsonUtils.writeAsString(jobContext.getCommandDTO())); + stageContext.setServiceDTO(serviceDTO); + stageContext.setComponentDTO(componentDTO); + stageContext.setStackName(stackName); + stageContext.setStackVersion(stackVersion); + stageContext.setHostnames(hostnames); + + return stageContext; + } + + protected List getTodoListForCommand(Command command) { + try { + List orderedList = dag.topologicalSort(); + List componentNames = getComponentNames(); + List componentCommandNames = new ArrayList<>(componentNames.stream().map(x -> x + "-" + command.name().toUpperCase()).toList()); + + orderedList.retainAll(componentCommandNames); + componentCommandNames.removeAll(orderedList); + orderedList.addAll(componentCommandNames); + + return orderedList; + } catch (Exception e) { + throw new ServerException(e); + } + } + + protected List getComponentNames() { + List serviceNames = getServiceNames(); + List components = componentRepository.findAllByClusterIdAndServiceServiceNameIn(cluster.getId(), serviceNames); + + return components.stream().map(Component::getComponentName).toList(); + } + + protected String findServiceNameByComponentName(String componentName) { + for (ServiceDTO serviceDTO : serviceNameToDTO.values()) { + for (ComponentDTO componentDTO : serviceDTO.getComponents()) { + if (componentDTO.getComponentName().equals(componentName)) { + return serviceDTO.getServiceName(); + } + } } - return componentHostMapping; + return null; } - protected List getComponents() { - Long clusterId = jobContext.getCommandDTO().getClusterId(); - List serviceNameList = jobContext.getCommandDTO().getServiceCommands() - .stream() - .map(ServiceCommandDTO::getServiceName) - .toList(); + protected Boolean isMasterComponent(String componentName) { + ComponentDTO componentDTO = componentNameToDTO.get(componentName); + return componentDTO.getCategory().equalsIgnoreCase("master"); + } + + protected List findHostnamesByComponentName(String componentName) { + List hostComponents = hostComponentRepository.findAllByComponentClusterIdAndComponentComponentName(cluster.getId(), componentName); + if (hostComponents == null) { + return new ArrayList<>(); + } else { + return hostComponents.stream().map(HostComponent::getHost).map(Host::getHostname).toList(); + } + } + + protected void createCacheStage() { + StageContext stageContext = StageContext.fromPayload(JsonUtils.writeAsString(jobContext.getCommandDTO())); + stageContext.setStackName(stackName); + stageContext.setStackVersion(stackVersion); + stages.add(StageFactories.getStageFactory(StageType.HOST_CACHE).createStage(stageContext)); + } + + protected void createInstallStages() { + List todoList = getTodoListForCommand(Command.INSTALL); + + for (String componentCommand : todoList) { + String[] split = componentCommand.split("-"); + String componentName = split[0]; + String serviceName = findServiceNameByComponentName(componentName); + + StageContext stageContext = createStageContext(serviceName, componentName); + stages.add(StageFactories.getStageFactory(StageType.COMPONENT_INSTALL).createStage(stageContext)); + } + } + + protected void createStartStages() { + List todoList = getTodoListForCommand(Command.START); + + for (String componentCommand : todoList) { + String[] split = componentCommand.split("-"); + String componentName = split[0]; + String serviceName = findServiceNameByComponentName(componentName); + + if (!isMasterComponent(componentName)) { + continue; + } + + StageContext stageContext = createStageContext(serviceName, componentName); + stages.add(StageFactories.getStageFactory(StageType.COMPONENT_START).createStage(stageContext)); + } + } + + protected void createStopStages() { + List todoList = getTodoListForCommand(Command.STOP); + + for (String componentCommand : todoList) { + String[] split = componentCommand.split("-"); + String componentName = split[0]; + String serviceName = findServiceNameByComponentName(componentName); + + if (!isMasterComponent(componentName)) { + continue; + } + + StageContext stageContext = createStageContext(serviceName, componentName); + stages.add(StageFactories.getStageFactory(StageType.COMPONENT_STOP).createStage(stageContext)); + } + } + + protected void createCheckStages() { + List todoList = getTodoListForCommand(Command.CHECK); + + for (String componentCommand : todoList) { + String[] split = componentCommand.split("-"); + String componentName = split[0]; + String serviceName = findServiceNameByComponentName(componentName); + List hostnames = findHostnamesByComponentName(componentName); + + if (!isMasterComponent(componentName)) { + continue; + } + + StageContext stageContext = createStageContext(serviceName, componentName, List.of(hostnames.get(0))); + stages.add(StageFactories.getStageFactory(StageType.COMPONENT_CHECK).createStage(stageContext)); + } + } - return componentRepository.findAllByClusterIdAndServiceServiceNameIn(clusterId, serviceNameList); + private List getServiceNames() { + return jobContext.getCommandDTO().getServiceCommands().stream().map(ServiceCommandDTO::getServiceName).toList(); } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/ServiceConfigureJobFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/ServiceConfigureJobFactory.java index f33316ce..c2b43bad 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/ServiceConfigureJobFactory.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/ServiceConfigureJobFactory.java @@ -1,37 +1,16 @@ package org.apache.bigtop.manager.server.command.job.factory.service; -import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.apache.bigtop.manager.common.enums.Command; -import org.apache.bigtop.manager.common.utils.JsonUtils; import org.apache.bigtop.manager.server.command.CommandIdentifier; -import org.apache.bigtop.manager.server.command.job.helper.HostCacheStageHelper; -import org.apache.bigtop.manager.server.command.strategy.StageCallback; import org.apache.bigtop.manager.server.enums.CommandLevel; -import org.apache.bigtop.manager.server.model.dto.CommandDTO; -import org.apache.bigtop.manager.server.model.dto.ServiceConfigDTO; -import org.apache.bigtop.manager.server.model.dto.command.ServiceCommandDTO; -import org.apache.bigtop.manager.server.orm.entity.Cluster; -import org.apache.bigtop.manager.server.orm.entity.Stage; -import org.apache.bigtop.manager.server.orm.repository.ClusterRepository; -import org.apache.bigtop.manager.server.service.ConfigService; -import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Scope; @Slf4j @org.springframework.stereotype.Component @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) -public class ServiceConfigureJobFactory extends AbstractServiceJobFactory implements StageCallback { - - @Resource - private HostCacheStageHelper hostCacheStageHelper; - - @Resource - private ClusterRepository clusterRepository; - - @Resource - private ConfigService configService; +public class ServiceConfigureJobFactory extends AbstractServiceJobFactory { @Override public CommandIdentifier getCommandIdentifier() { @@ -40,20 +19,8 @@ public class ServiceConfigureJobFactory extends AbstractServiceJobFactory implem @Override public void createStagesAndTasks() { - String callbackClassName = this.getClass().getName(); - String payload = JsonUtils.writeAsString(jobContext.getCommandDTO()); - stages.add(hostCacheStageHelper.createStage(cluster.getId(), callbackClassName, payload)); - } - - @Override - public void beforeStage(Stage stage) { - CommandDTO commandDTO = JsonUtils.readFromString(stage.getPayload(), CommandDTO.class); + super.initAttrs(); - Cluster cluster = clusterRepository.getReferenceById(commandDTO.getClusterId()); - for (ServiceCommandDTO serviceCommand : commandDTO.getServiceCommands()) { - ServiceConfigDTO serviceConfigDTO = new ServiceConfigDTO(); - BeanUtils.copyProperties(serviceCommand, serviceConfigDTO); - configService.updateConfig(cluster, serviceConfigDTO); - } + super.createCacheStage(); } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/ServiceInstallJobFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/ServiceInstallJobFactory.java index dba957a7..4483b658 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/ServiceInstallJobFactory.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/ServiceInstallJobFactory.java @@ -1,54 +1,21 @@ package org.apache.bigtop.manager.server.command.job.factory.service; -import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.apache.bigtop.manager.common.enums.Command; -import org.apache.bigtop.manager.common.message.type.RequestMessage; -import org.apache.bigtop.manager.common.utils.JsonUtils; -import org.apache.bigtop.manager.server.enums.CommandLevel; -import org.apache.bigtop.manager.server.enums.JobState; -import org.apache.bigtop.manager.server.enums.MaintainState; import org.apache.bigtop.manager.server.command.CommandIdentifier; -import org.apache.bigtop.manager.server.command.job.helper.HostCacheStageHelper; -import org.apache.bigtop.manager.server.command.strategy.StageCallback; -import org.apache.bigtop.manager.server.model.dto.*; +import org.apache.bigtop.manager.server.enums.CommandLevel; +import org.apache.bigtop.manager.server.model.dto.ComponentHostDTO; import org.apache.bigtop.manager.server.model.dto.command.ServiceCommandDTO; -import org.apache.bigtop.manager.server.model.mapper.ComponentMapper; -import org.apache.bigtop.manager.server.model.mapper.ServiceMapper; -import org.apache.bigtop.manager.server.orm.entity.*; -import org.apache.bigtop.manager.server.orm.repository.HostComponentRepository; -import org.apache.bigtop.manager.server.orm.repository.ServiceRepository; -import org.apache.bigtop.manager.server.service.ServiceService; -import org.apache.bigtop.manager.server.utils.StackUtils; -import org.apache.commons.lang3.tuple.ImmutablePair; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Scope; import java.util.ArrayList; import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.function.Function; -import java.util.stream.Collectors; - -import static org.apache.bigtop.manager.common.constants.Constants.CACHE_STAGE_NAME; @Slf4j @org.springframework.stereotype.Component @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) -public class ServiceInstallJobFactory extends AbstractServiceJobFactory implements StageCallback { - - @Resource - private ServiceRepository serviceRepository; - - @Resource - private HostComponentRepository hostComponentRepository; - - @Resource - private ServiceService serviceService; - - @Resource - private HostCacheStageHelper hostCacheStageHelper; +public class ServiceInstallJobFactory extends AbstractServiceJobFactory { @Override public CommandIdentifier getCommandIdentifier() { @@ -60,112 +27,46 @@ public class ServiceInstallJobFactory extends AbstractServiceJobFactory implemen */ @Override protected void createStagesAndTasks() { - // Install all required components in service - super.createStagesAndTasks(); - - String callbackClassName = this.getClass().getName(); + super.initAttrs(); - // cache stage - String payload = JsonUtils.writeAsString(jobContext.getCommandDTO()); - stages.add(hostCacheStageHelper.createStage(cluster.getId(), callbackClassName, payload)); + // Install components + super.createInstallStages(); - // Start all required components in service - createStages(callbackClassName, Command.START); + // Cache configs when after installed + super.createCacheStage(); - // Check all required components in service - createStages(callbackClassName, Command.CHECK); + // Start all master components + super.createStartStages(); - // StageLifecycles.getStageLifecycle(StageType.COMPONENT_INSTALL).createStage(stageLifecycleContext); - // StageLifecycles.getStageLifecycle(StageType.COMPONENT_INSTALL).beforeStage(stageLifecycleContext); - // StageLifecycles.getStageLifecycle(StageType.COMPONENT_INSTALL).onStageSuccess(stageLifecycleContext); - // StageLifecycles.getStageLifecycle(StageType.COMPONENT_INSTALL).onStageFailed(stageLifecycleContext); - - } - - @Override - protected Map> getComponentHostMapping(Command command) { - return jobContext.getCommandDTO().getServiceCommands().stream() - .flatMap(serviceCommand -> serviceCommand.getComponentHosts().stream()) - .collect(Collectors.toMap(ComponentHostDTO::getComponentName, componentHostDTO -> { - if (command == Command.CHECK) { - return List.of(componentHostDTO.getHostnames().get(0)); - } else { - return componentHostDTO.getHostnames(); - } - })); + // Check all master components after started + super.createCheckStages(); } @Override - protected List getComponents() { - List components = new ArrayList<>(); - CommandDTO commandDTO = jobContext.getCommandDTO(); - Long clusterId = commandDTO.getClusterId(); - - String stackName = cluster.getStack().getStackName(); - String stackVersion = cluster.getStack().getStackVersion(); - - Map>> stackKeyMap = StackUtils.getStackKeyMap(); - - ImmutablePair> immutablePair = stackKeyMap.get(StackUtils.fullStackName(stackName, stackVersion)); - Map serviceNameToDTO = immutablePair.getRight() - .stream() - .collect(Collectors.toMap(ServiceDTO::getServiceName, Function.identity())); - - // Persist service, component and hostComponent metadata to database - for (ServiceCommandDTO serviceCommand : commandDTO.getServiceCommands()) { - String serviceName = serviceCommand.getServiceName(); - Optional serviceOptional = serviceRepository.findByClusterIdAndServiceName(clusterId, serviceName); - if (serviceOptional.isPresent()) { - continue; - } - - ServiceDTO serviceDTO = serviceNameToDTO.get(serviceName); - Service service = ServiceMapper.INSTANCE.fromDTO2Entity(serviceDTO, cluster); - List componentDTOList = serviceDTO.getComponents(); - for (ComponentDTO componentDTO : componentDTOList) { - Component component = ComponentMapper.INSTANCE.fromDTO2Entity(componentDTO, service, cluster); - components.add(component); + protected List getComponentNames() { + List componentNames = new ArrayList<>(); + for (ServiceCommandDTO serviceCommand : jobContext.getCommandDTO().getServiceCommands()) { + List componentHosts = serviceCommand.getComponentHosts(); + for (ComponentHostDTO componentHost : componentHosts) { + String componentName = componentHost.getComponentName(); + componentNames.add(componentName); } } - return components; - } - - @Override - public void beforeStage(Stage stage) { - CommandDTO commandDTO = JsonUtils.readFromString(stage.getPayload(), CommandDTO.class); - if (stage.getName().equals(CACHE_STAGE_NAME) && commandDTO.getCommand() == Command.INSTALL && commandDTO.getCommandLevel() == CommandLevel.SERVICE) { - serviceService.saveByCommand(commandDTO); - } - } - - @Override - public String generatePayload(Task task) { - Cluster cluster = task.getCluster(); - hostCacheStageHelper.createCache(cluster); - RequestMessage requestMessage = hostCacheStageHelper.getMessage(task.getHostname()); - log.info("[generatePayload]-[HostCacheJobFactory-requestMessage]: {}", requestMessage); - return JsonUtils.writeAsString(requestMessage); + return componentNames; } @Override - public void afterStage(Stage stage) { - Long clusterId = stage.getCluster().getId(); - String componentName = stage.getComponentName(); - CommandDTO commandDTO = JsonUtils.readFromString(stage.getPayload(), CommandDTO.class); - Command command = commandDTO.getCommand(); - - if (stage.getState() == JobState.SUCCESSFUL && command == Command.START) { - List hostComponents = hostComponentRepository.findAllByComponentClusterIdAndComponentComponentName(clusterId, componentName); - Service service = hostComponents.get(0).getComponent().getService(); - - hostComponents.forEach(hostComponent -> hostComponent.setState(MaintainState.STARTED)); - hostComponentRepository.saveAll(hostComponents); - - if (hostComponents.stream().allMatch(x -> x.getState() == MaintainState.STARTED)) { - service.setState(MaintainState.STARTED); + protected List findHostnamesByComponentName(String componentName) { + for (ServiceCommandDTO serviceCommand : jobContext.getCommandDTO().getServiceCommands()) { + List componentHosts = serviceCommand.getComponentHosts(); + for (ComponentHostDTO componentHost : componentHosts) { + if (componentHost.getComponentName().equals(componentName)) { + return componentHost.getHostnames(); + } } - serviceRepository.save(service); } + + return new ArrayList<>(); } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/ServiceRestartJobFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/ServiceRestartJobFactory.java index 1160b842..d5ec8d5a 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/ServiceRestartJobFactory.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/ServiceRestartJobFactory.java @@ -2,19 +2,27 @@ package org.apache.bigtop.manager.server.command.job.factory.service; import lombok.extern.slf4j.Slf4j; import org.apache.bigtop.manager.common.enums.Command; -import org.apache.bigtop.manager.server.enums.CommandLevel; import org.apache.bigtop.manager.server.command.CommandIdentifier; -import org.apache.bigtop.manager.server.command.strategy.StageCallback; +import org.apache.bigtop.manager.server.enums.CommandLevel; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Scope; @Slf4j @org.springframework.stereotype.Component @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) -public class ServiceRestartJobFactory extends AbstractServiceJobFactory implements StageCallback { +public class ServiceRestartJobFactory extends AbstractServiceJobFactory { @Override public CommandIdentifier getCommandIdentifier() { return new CommandIdentifier(CommandLevel.SERVICE, Command.RESTART); } + + @Override + protected void createStagesAndTasks() { + super.initAttrs(); + + super.createStopStages(); + + super.createStartStages(); + } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/ServiceStartJobFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/ServiceStartJobFactory.java index c9e80fb2..fdee2fd1 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/ServiceStartJobFactory.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/ServiceStartJobFactory.java @@ -1,33 +1,16 @@ package org.apache.bigtop.manager.server.command.job.factory.service; -import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.apache.bigtop.manager.common.enums.Command; -import org.apache.bigtop.manager.server.enums.CommandLevel; -import org.apache.bigtop.manager.server.enums.JobState; -import org.apache.bigtop.manager.server.enums.MaintainState; import org.apache.bigtop.manager.server.command.CommandIdentifier; -import org.apache.bigtop.manager.server.command.strategy.StageCallback; -import org.apache.bigtop.manager.server.orm.entity.HostComponent; -import org.apache.bigtop.manager.server.orm.entity.Service; -import org.apache.bigtop.manager.server.orm.entity.Stage; -import org.apache.bigtop.manager.server.orm.repository.HostComponentRepository; -import org.apache.bigtop.manager.server.orm.repository.ServiceRepository; +import org.apache.bigtop.manager.server.enums.CommandLevel; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Scope; -import java.util.List; - @Slf4j @org.springframework.stereotype.Component @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) -public class ServiceStartJobFactory extends AbstractServiceJobFactory implements StageCallback { - - @Resource - private ServiceRepository serviceRepository; - - @Resource - private HostComponentRepository hostComponentRepository; +public class ServiceStartJobFactory extends AbstractServiceJobFactory { @Override public CommandIdentifier getCommandIdentifier() { @@ -35,22 +18,9 @@ public class ServiceStartJobFactory extends AbstractServiceJobFactory implements } @Override - public void afterStage(Stage stage) { - Long clusterId = stage.getCluster().getId(); - String componentName = stage.getComponentName(); - - if (stage.getState() == JobState.SUCCESSFUL) { - List hostComponents = hostComponentRepository.findAllByComponentClusterIdAndComponentComponentName(clusterId, componentName); - Service service = hostComponents.get(0).getComponent().getService(); + protected void createStagesAndTasks() { + super.initAttrs(); - hostComponents.forEach(hostComponent -> hostComponent.setState(MaintainState.STARTED)); - hostComponentRepository.saveAll(hostComponents); - - if (hostComponents.stream().allMatch(x -> x.getState() == MaintainState.STARTED)) { - service.setState(MaintainState.STARTED); - } - serviceRepository.save(service); - } + super.createStartStages(); } - } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/ServiceStopJobFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/ServiceStopJobFactory.java index 1592edb6..66b4e440 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/ServiceStopJobFactory.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/factory/service/ServiceStopJobFactory.java @@ -1,33 +1,16 @@ package org.apache.bigtop.manager.server.command.job.factory.service; -import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.apache.bigtop.manager.common.enums.Command; -import org.apache.bigtop.manager.server.enums.CommandLevel; -import org.apache.bigtop.manager.server.enums.JobState; -import org.apache.bigtop.manager.server.enums.MaintainState; import org.apache.bigtop.manager.server.command.CommandIdentifier; -import org.apache.bigtop.manager.server.command.strategy.StageCallback; -import org.apache.bigtop.manager.server.orm.entity.HostComponent; -import org.apache.bigtop.manager.server.orm.entity.Service; -import org.apache.bigtop.manager.server.orm.entity.Stage; -import org.apache.bigtop.manager.server.orm.repository.HostComponentRepository; -import org.apache.bigtop.manager.server.orm.repository.ServiceRepository; +import org.apache.bigtop.manager.server.enums.CommandLevel; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Scope; -import java.util.List; - @Slf4j @org.springframework.stereotype.Component @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) -public class ServiceStopJobFactory extends AbstractServiceJobFactory implements StageCallback { - - @Resource - private ServiceRepository serviceRepository; - - @Resource - private HostComponentRepository hostComponentRepository; +public class ServiceStopJobFactory extends AbstractServiceJobFactory { @Override public CommandIdentifier getCommandIdentifier() { @@ -35,22 +18,9 @@ public class ServiceStopJobFactory extends AbstractServiceJobFactory implements } @Override - public void afterStage(Stage stage) { - Long clusterId = stage.getCluster().getId(); - String componentName = stage.getComponentName(); - - if (stage.getState() == JobState.SUCCESSFUL) { - List hostComponents = hostComponentRepository.findAllByComponentClusterIdAndComponentComponentName(clusterId, componentName); - Service service = hostComponents.get(0).getComponent().getService(); + protected void createStagesAndTasks() { + super.initAttrs(); - hostComponents.forEach(hostComponent -> hostComponent.setState(MaintainState.STOPPED)); - hostComponentRepository.saveAll(hostComponents); - - if (hostComponents.stream().allMatch(x -> x.getState() == MaintainState.STOPPED)) { - service.setState(MaintainState.STOPPED); - } - serviceRepository.save(service); - } + super.createStopStages(); } - } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/helper/HostCacheStageHelper.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/helper/HostCacheStageHelper.java deleted file mode 100644 index ad68666f..00000000 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/helper/HostCacheStageHelper.java +++ /dev/null @@ -1,269 +0,0 @@ -package org.apache.bigtop.manager.server.command.job.helper; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.google.common.collect.Sets; -import jakarta.annotation.Resource; -import lombok.extern.slf4j.Slf4j; -import org.apache.bigtop.manager.common.enums.Command; -import org.apache.bigtop.manager.common.enums.MessageType; -import org.apache.bigtop.manager.common.message.type.HostCachePayload; -import org.apache.bigtop.manager.common.message.type.RequestMessage; -import org.apache.bigtop.manager.common.message.type.pojo.ClusterInfo; -import org.apache.bigtop.manager.common.message.type.pojo.ComponentInfo; -import org.apache.bigtop.manager.common.message.type.pojo.RepoInfo; -import org.apache.bigtop.manager.common.utils.JsonUtils; -import org.apache.bigtop.manager.server.enums.JobState; -import org.apache.bigtop.manager.server.model.dto.PropertyDTO; -import org.apache.bigtop.manager.server.model.mapper.RepoMapper; -import org.apache.bigtop.manager.server.orm.entity.*; -import org.apache.bigtop.manager.server.orm.repository.*; -import org.apache.bigtop.manager.server.utils.StackConfigUtils; -import org.apache.commons.lang3.StringUtils; - -import java.util.*; -import java.util.stream.Collectors; - -import static org.apache.bigtop.manager.common.constants.Constants.ALL_HOST_KEY; -import static org.apache.bigtop.manager.common.constants.Constants.CACHE_STAGE_NAME; - -@Slf4j -@org.springframework.stereotype.Component -public class HostCacheStageHelper { - - @Resource - private ClusterRepository clusterRepository; - - @Resource - private HostComponentRepository hostComponentRepository; - - @Resource - private ServiceRepository serviceRepository; - - @Resource - private ServiceConfigMappingRepository serviceConfigMappingRepository; - - @Resource - private RepoRepository repoRepository; - - @Resource - private SettingRepository settingRepository; - - @Resource - private HostRepository hostRepository; - - @Resource - private ComponentRepository componentRepository; - - public Stage createStage(Long clusterId, String callbackClassName, String payload) { - Cluster cluster = clusterRepository.getReferenceById(clusterId); - createCache(cluster); - - List hostList = hostRepository.findAllByClusterId(clusterId); - List hostnames = hostList.stream().map(Host::getHostname).toList(); - - Stage hostCacheStage = new Stage(); - hostCacheStage.setName(CACHE_STAGE_NAME); - - if (StringUtils.isNotEmpty(callbackClassName)) { - hostCacheStage.setCallbackClassName(callbackClassName); - } else { - hostCacheStage.setCallbackClassName(this.getClass().getName()); - } - - if (StringUtils.isNotEmpty(callbackClassName)) { - hostCacheStage.setPayload(payload); - } - - List tasks = new ArrayList<>(); - for (String hostname : hostnames) { - Task task = new Task(); - task.setName("Cache host for " + hostname); - task.setStackName(cluster.getStack().getStackName()); - task.setStackVersion(cluster.getStack().getStackVersion()); - task.setHostname(hostname); - task.setServiceName("cluster"); - task.setServiceUser("root"); - task.setServiceGroup("root"); - task.setComponentName("bigtop-manager-agent"); - task.setCommand(Command.CUSTOM_COMMAND); - task.setCustomCommand("cache_host"); - - RequestMessage requestMessage = getMessage(hostname); - log.info("[HostCacheJobFactory-requestMessage]: {}", requestMessage); - task.setContent(JsonUtils.writeAsString(requestMessage)); - task.setMessageId(requestMessage.getMessageId()); - - tasks.add(task); - } - - hostCacheStage.setTasks(tasks); - return hostCacheStage; - } - - private ClusterInfo clusterInfo; - - private Map componentInfoMap; - - private Map> serviceConfigMap; - - private Map> hostMap; - - private List repoList; - - private Map> userMap; - - private Map settingsMap; - - public void createCache(Cluster cluster) { - Long clusterId = cluster.getId(); - - String clusterName = cluster.getClusterName(); - String stackName = cluster.getStack().getStackName(); - String stackVersion = cluster.getStack().getStackVersion(); - - List services = serviceRepository.findAllByClusterId(clusterId); - List serviceConfigMappingList = serviceConfigMappingRepository.findAllGroupLastest(clusterId); - List hostComponents = hostComponentRepository.findAllByComponentClusterId(clusterId); - List repos = repoRepository.findAllByCluster(cluster); - Iterable settings = settingRepository.findAll(); - List hostList = hostRepository.findAllByClusterId(clusterId); - - //Wrapper clusterInfo for HostCacheMessage - clusterInfo = new ClusterInfo(); - clusterInfo.setClusterName(clusterName); - clusterInfo.setStackName(stackName); - clusterInfo.setStackVersion(stackVersion); - clusterInfo.setUserGroup(cluster.getUserGroup()); - clusterInfo.setRepoTemplate(cluster.getRepoTemplate()); - clusterInfo.setRoot(cluster.getRoot()); - - try { - Set packages = Sets.newHashSet(cluster.getPackages().split(",")); - clusterInfo.setPackages(packages); - } catch (Exception e) { - log.warn("no packages"); - } - - //Wrapper serviceConfigMap for HostCacheMessage - serviceConfigMap = new HashMap<>(); - serviceConfigMappingList.forEach(scm -> { - ServiceConfig sc = scm.getServiceConfig(); - List properties = JsonUtils.readFromString(sc.getPropertiesJson(), new TypeReference<>() { - }); - String configMapStr = JsonUtils.writeAsString(StackConfigUtils.extractConfigMap(properties)); - - if (serviceConfigMap.containsKey(sc.getService().getServiceName())) { - serviceConfigMap.get(sc.getService().getServiceName()).put(sc.getTypeName(), configMapStr); - } else { - Map hashMap = new HashMap<>(); - hashMap.put(sc.getTypeName(), configMapStr); - serviceConfigMap.put(sc.getService().getServiceName(), hashMap); - } - }); - - //Wrapper hostMap for HostCacheMessage - hostMap = new HashMap<>(); - hostComponents.forEach(x -> { - if (hostMap.containsKey(x.getComponent().getComponentName())) { - hostMap.get(x.getComponent().getComponentName()).add(x.getHost().getHostname()); - } else { - Set set = new HashSet<>(); - set.add(x.getHost().getHostname()); - hostMap.put(x.getComponent().getComponentName(), set); - } - hostMap.get(x.getComponent().getComponentName()).add(x.getHost().getHostname()); - }); - - Set hostNameSet = hostList.stream().map(Host::getHostname).collect(Collectors.toSet()); - hostMap.put(ALL_HOST_KEY, hostNameSet); - - //Wrapper repoList for HostCacheMessage - repoList = new ArrayList<>(); - repos.forEach(repo -> { - RepoInfo repoInfo = RepoMapper.INSTANCE.fromEntity2Message(repo); - repoList.add(repoInfo); - }); - - //Wrapper userMap for HostCacheMessage - userMap = new HashMap<>(); - services.forEach(x -> userMap.put(x.getServiceUser(), Set.of(x.getServiceGroup()))); - - //Wrapper settings for HostCacheMessage - settingsMap = new HashMap<>(); - settings.forEach(x -> settingsMap.put(x.getTypeName(), x.getConfigData())); - - //Wrapper componentInfoList for HostCacheMessage - componentInfoMap = new HashMap<>(); - List componentList = componentRepository.findAll(); - componentList.forEach(c -> { - ComponentInfo componentInfo = new ComponentInfo(); - componentInfo.setComponentName(c.getComponentName()); - componentInfo.setCommandScript(c.getCommandScript()); - componentInfo.setDisplayName(c.getDisplayName()); - componentInfo.setCategory(c.getCategory()); - componentInfo.setCustomCommands(c.getCustomCommands()); - componentInfo.setServiceName(c.getService().getServiceName()); - componentInfoMap.put(c.getComponentName(), componentInfo); - }); - } - - public RequestMessage getMessage(String hostname) { - return getMessage(hostname, - settingsMap, - clusterInfo, - serviceConfigMap, - hostMap, - repoList, - userMap, - componentInfoMap); - } - - public RequestMessage getMessage(String hostname, - Map settingsMap, - ClusterInfo clusterInfo, - Map> serviceConfigMap, - Map> hostMap, - List repoList, - Map> userMap, - Map componentInfoMap) { - HostCachePayload hostCachePayload = getMessagePayload( - hostname, - settingsMap, - clusterInfo, - serviceConfigMap, - hostMap, - repoList, - userMap, - componentInfoMap); - RequestMessage requestMessage = new RequestMessage(); - - requestMessage.setMessageType(MessageType.HOST_CACHE); - requestMessage.setHostname(hostname); - - requestMessage.setMessagePayload(JsonUtils.writeAsString(hostCachePayload)); - return requestMessage; - } - - private HostCachePayload getMessagePayload( - String hostname, - Map settingsMap, - ClusterInfo clusterInfo, - Map> serviceConfigMap, - Map> hostMap, - List repoList, - Map> userMap, - Map componentInfoMap) { - HostCachePayload hostCacheMessage = new HostCachePayload(); - hostCacheMessage.setHostname(hostname); - - hostCacheMessage.setClusterInfo(clusterInfo); - hostCacheMessage.setConfigurations(serviceConfigMap); - hostCacheMessage.setClusterHostInfo(hostMap); - hostCacheMessage.setRepoInfo(repoList); - hostCacheMessage.setSettings(settingsMap); - hostCacheMessage.setUserInfo(userMap); - hostCacheMessage.setComponentInfo(componentInfoMap); - return hostCacheMessage; - } - -} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/helper/HostCheckStageHelper.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/helper/HostCheckStageHelper.java deleted file mode 100644 index b248f2d1..00000000 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/helper/HostCheckStageHelper.java +++ /dev/null @@ -1,86 +0,0 @@ -package org.apache.bigtop.manager.server.command.job.helper; - -import jakarta.annotation.Resource; -import lombok.extern.slf4j.Slf4j; -import org.apache.bigtop.manager.common.enums.Command; -import org.apache.bigtop.manager.common.enums.MessageType; -import org.apache.bigtop.manager.common.message.type.HostCheckPayload; -import org.apache.bigtop.manager.common.message.type.RequestMessage; -import org.apache.bigtop.manager.common.message.type.pojo.HostCheckType; -import org.apache.bigtop.manager.common.utils.JsonUtils; -import org.apache.bigtop.manager.server.enums.JobState; -import org.apache.bigtop.manager.server.orm.entity.Cluster; -import org.apache.bigtop.manager.server.orm.entity.Job; -import org.apache.bigtop.manager.server.orm.entity.Stage; -import org.apache.bigtop.manager.server.orm.entity.Task; -import org.apache.bigtop.manager.server.orm.repository.ClusterRepository; -import org.apache.bigtop.manager.server.orm.repository.StageRepository; -import org.apache.bigtop.manager.server.orm.repository.TaskRepository; - -import java.util.ArrayList; -import java.util.List; - -@Slf4j -@org.springframework.stereotype.Component -public class HostCheckStageHelper { - - @Resource - private ClusterRepository clusterRepository; - - public Stage createStage(Long clusterId, List hostnames, String callbackClassName) { - Cluster cluster = clusterRepository.getReferenceById(clusterId); - - String stackName = cluster.getStack().getStackName(); - String stackVersion = cluster.getStack().getStackVersion(); - return createStage(stackName, stackVersion, hostnames, callbackClassName); - } - - public Stage createStage(String stackName, String stackVersion, List hostnames, String callbackClassName) { - // Create stages - Stage hostCheckStage = new Stage(); - hostCheckStage.setName("Check Hosts"); - hostCheckStage.setCallbackClassName(callbackClassName); - - List tasks = new ArrayList<>(); - for (String hostname : hostnames) { - Task task = new Task(); - task.setName("Check host for " + hostname); - task.setStackName(stackName); - task.setStackVersion(stackVersion); - task.setHostname(hostname); - task.setServiceName("cluster"); - task.setServiceUser("root"); - task.setServiceGroup("root"); - task.setComponentName("bigtop-manager-agent"); - task.setCommand(Command.CUSTOM_COMMAND); - task.setCustomCommand("check_host"); - - RequestMessage requestMessage = createMessage(hostname); - task.setContent(JsonUtils.writeAsString(requestMessage)); - task.setMessageId(requestMessage.getMessageId()); - - tasks.add(task); - } - - hostCheckStage.setTasks(tasks); - return hostCheckStage; - } - - public RequestMessage createMessage(String hostname) { - HostCheckPayload hostCheckPayload = createMessagePayload(hostname); - RequestMessage requestMessage = new RequestMessage(); - - requestMessage.setMessageType(MessageType.HOST_CHECK); - requestMessage.setHostname(hostname); - - requestMessage.setMessagePayload(JsonUtils.writeAsString(hostCheckPayload)); - return requestMessage; - } - - public HostCheckPayload createMessagePayload(String hostname) { - HostCheckPayload hostCheckMessage = new HostCheckPayload(); - hostCheckMessage.setHostCheckTypes(HostCheckType.values()); - hostCheckMessage.setHostname(hostname); - return hostCheckMessage; - } -} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/AbstractJobRunner.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/AbstractJobRunner.java index 721db9cb..38613f5c 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/AbstractJobRunner.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/AbstractJobRunner.java @@ -1,9 +1,11 @@ package org.apache.bigtop.manager.server.command.job.runner; import jakarta.annotation.Resource; +import org.apache.bigtop.manager.common.utils.JsonUtils; import org.apache.bigtop.manager.server.command.stage.runner.StageRunner; import org.apache.bigtop.manager.server.command.stage.runner.StageRunners; import org.apache.bigtop.manager.server.enums.JobState; +import org.apache.bigtop.manager.server.model.dto.CommandDTO; import org.apache.bigtop.manager.server.orm.entity.Job; import org.apache.bigtop.manager.server.orm.entity.Stage; import org.apache.bigtop.manager.server.orm.entity.Task; @@ -90,4 +92,8 @@ public abstract class AbstractJobRunner implements JobRunner { job.setState(JobState.FAILED); jobRepository.save(job); } + + protected CommandDTO getCommandDTO() { + return JsonUtils.readFromString(job.getPayload(), CommandDTO.class); + } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/cluster/ClusterCreateJobRunner.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/cluster/ClusterCreateJobRunner.java index a89919ed..c15bc01a 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/cluster/ClusterCreateJobRunner.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/cluster/ClusterCreateJobRunner.java @@ -8,9 +8,13 @@ import org.apache.bigtop.manager.server.command.CommandIdentifier; import org.apache.bigtop.manager.server.command.job.runner.AbstractJobRunner; import org.apache.bigtop.manager.server.enums.CommandLevel; import org.apache.bigtop.manager.server.model.dto.ClusterDTO; -import org.apache.bigtop.manager.server.model.dto.command.ClusterCommandDTO; +import org.apache.bigtop.manager.server.model.dto.CommandDTO; +import org.apache.bigtop.manager.server.model.mapper.ClusterMapper; +import org.apache.bigtop.manager.server.orm.entity.Cluster; +import org.apache.bigtop.manager.server.orm.entity.Stage; +import org.apache.bigtop.manager.server.orm.entity.Task; +import org.apache.bigtop.manager.server.orm.repository.*; import org.apache.bigtop.manager.server.service.ClusterService; -import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Scope; @@ -22,6 +26,18 @@ public class ClusterCreateJobRunner extends AbstractJobRunner { @Resource private ClusterService clusterService; + @Resource + private ClusterRepository clusterRepository; + + @Resource + private JobRepository jobRepository; + + @Resource + private StageRepository stageRepository; + + @Resource + private TaskRepository taskRepository; + @Override public CommandIdentifier getCommandIdentifier() { return new CommandIdentifier(CommandLevel.CLUSTER, Command.CREATE); @@ -29,10 +45,26 @@ public class ClusterCreateJobRunner extends AbstractJobRunner { @Override public void onSuccess() { - ClusterCommandDTO clusterCommand = JsonUtils.readFromString(job.getPayload(), ClusterCommandDTO.class); - ClusterDTO clusterDTO = new ClusterDTO(); - BeanUtils.copyProperties(clusterCommand, clusterDTO); + super.onSuccess(); + // Save Cluster + CommandDTO commandDTO = JsonUtils.readFromString(job.getPayload(), CommandDTO.class); + ClusterDTO clusterDTO = ClusterMapper.INSTANCE.fromCommand2DTO(commandDTO.getClusterCommand()); clusterService.save(clusterDTO); + + // Link job to cluster after cluster successfully added + Cluster cluster = clusterRepository.findByClusterName(commandDTO.getClusterCommand().getClusterName()).orElse(new Cluster()); + job.setCluster(cluster); + jobRepository.save(job); + + for (Stage stage : job.getStages()) { + stage.setCluster(cluster); + stageRepository.save(stage); + + for (Task task : stage.getTasks()) { + task.setCluster(cluster); + taskRepository.save(task); + } + } } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/component/ComponentInstallJobRunner.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/component/ComponentInstallJobRunner.java new file mode 100644 index 00000000..525bbc71 --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/component/ComponentInstallJobRunner.java @@ -0,0 +1,20 @@ +package org.apache.bigtop.manager.server.command.job.runner.component; + +import lombok.extern.slf4j.Slf4j; +import org.apache.bigtop.manager.common.enums.Command; +import org.apache.bigtop.manager.server.command.CommandIdentifier; +import org.apache.bigtop.manager.server.command.job.runner.AbstractJobRunner; +import org.apache.bigtop.manager.server.enums.CommandLevel; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; + +@Slf4j +@org.springframework.stereotype.Component +@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) +public class ComponentInstallJobRunner extends AbstractJobRunner { + + @Override + public CommandIdentifier getCommandIdentifier() { + return new CommandIdentifier(CommandLevel.COMPONENT, Command.INSTALL); + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/component/ComponentRestartJobRunner.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/component/ComponentRestartJobRunner.java new file mode 100644 index 00000000..3b8887de --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/component/ComponentRestartJobRunner.java @@ -0,0 +1,20 @@ +package org.apache.bigtop.manager.server.command.job.runner.component; + +import lombok.extern.slf4j.Slf4j; +import org.apache.bigtop.manager.common.enums.Command; +import org.apache.bigtop.manager.server.command.CommandIdentifier; +import org.apache.bigtop.manager.server.command.job.runner.AbstractJobRunner; +import org.apache.bigtop.manager.server.enums.CommandLevel; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; + +@Slf4j +@org.springframework.stereotype.Component +@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) +public class ComponentRestartJobRunner extends AbstractJobRunner { + + @Override + public CommandIdentifier getCommandIdentifier() { + return new CommandIdentifier(CommandLevel.COMPONENT, Command.RESTART); + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/component/ComponentStartJobRunner.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/component/ComponentStartJobRunner.java new file mode 100644 index 00000000..4dd46108 --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/component/ComponentStartJobRunner.java @@ -0,0 +1,20 @@ +package org.apache.bigtop.manager.server.command.job.runner.component; + +import lombok.extern.slf4j.Slf4j; +import org.apache.bigtop.manager.common.enums.Command; +import org.apache.bigtop.manager.server.command.CommandIdentifier; +import org.apache.bigtop.manager.server.command.job.runner.AbstractJobRunner; +import org.apache.bigtop.manager.server.enums.CommandLevel; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; + +@Slf4j +@org.springframework.stereotype.Component +@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) +public class ComponentStartJobRunner extends AbstractJobRunner { + + @Override + public CommandIdentifier getCommandIdentifier() { + return new CommandIdentifier(CommandLevel.COMPONENT, Command.START); + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/component/ComponentStopJobRunner.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/component/ComponentStopJobRunner.java new file mode 100644 index 00000000..cf4b0bc4 --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/component/ComponentStopJobRunner.java @@ -0,0 +1,20 @@ +package org.apache.bigtop.manager.server.command.job.runner.component; + +import lombok.extern.slf4j.Slf4j; +import org.apache.bigtop.manager.common.enums.Command; +import org.apache.bigtop.manager.server.command.CommandIdentifier; +import org.apache.bigtop.manager.server.command.job.runner.AbstractJobRunner; +import org.apache.bigtop.manager.server.enums.CommandLevel; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; + +@Slf4j +@org.springframework.stereotype.Component +@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) +public class ComponentStopJobRunner extends AbstractJobRunner { + + @Override + public CommandIdentifier getCommandIdentifier() { + return new CommandIdentifier(CommandLevel.COMPONENT, Command.STOP); + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/host/HostAddJobRunner.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/host/HostAddJobRunner.java new file mode 100644 index 00000000..4b0dd6c4 --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/host/HostAddJobRunner.java @@ -0,0 +1,41 @@ +package org.apache.bigtop.manager.server.command.job.runner.host; + +import jakarta.annotation.Resource; +import lombok.extern.slf4j.Slf4j; +import org.apache.bigtop.manager.common.enums.Command; +import org.apache.bigtop.manager.common.utils.JsonUtils; +import org.apache.bigtop.manager.server.command.CommandIdentifier; +import org.apache.bigtop.manager.server.command.job.runner.AbstractJobRunner; +import org.apache.bigtop.manager.server.enums.CommandLevel; +import org.apache.bigtop.manager.server.model.dto.CommandDTO; +import org.apache.bigtop.manager.server.model.dto.command.HostCommandDTO; +import org.apache.bigtop.manager.server.service.HostService; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; + +import java.util.List; + +@Slf4j +@org.springframework.stereotype.Component +@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) +public class HostAddJobRunner extends AbstractJobRunner { + + @Resource + private HostService hostService; + + @Override + public CommandIdentifier getCommandIdentifier() { + return new CommandIdentifier(CommandLevel.HOST, Command.INSTALL); + } + + @Override + public void onSuccess() { + super.onSuccess(); + + CommandDTO commandDTO = JsonUtils.readFromString(job.getPayload(), CommandDTO.class); + List hostCommands = commandDTO.getHostCommands(); + + List hostnames = hostCommands.stream().map(HostCommandDTO::getHostname).toList(); + hostService.batchSave(job.getCluster().getId(), hostnames); + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/service/ServiceConfigureJobRunner.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/service/ServiceConfigureJobRunner.java new file mode 100644 index 00000000..003ecdc2 --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/service/ServiceConfigureJobRunner.java @@ -0,0 +1,20 @@ +package org.apache.bigtop.manager.server.command.job.runner.service; + +import lombok.extern.slf4j.Slf4j; +import org.apache.bigtop.manager.common.enums.Command; +import org.apache.bigtop.manager.server.command.CommandIdentifier; +import org.apache.bigtop.manager.server.command.job.runner.AbstractJobRunner; +import org.apache.bigtop.manager.server.enums.CommandLevel; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; + +@Slf4j +@org.springframework.stereotype.Component +@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) +public class ServiceConfigureJobRunner extends AbstractJobRunner { + + @Override + public CommandIdentifier getCommandIdentifier() { + return new CommandIdentifier(CommandLevel.SERVICE, Command.CONFIGURE); + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/service/ServiceInstallJobRunner.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/service/ServiceInstallJobRunner.java new file mode 100644 index 00000000..6d9118c3 --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/service/ServiceInstallJobRunner.java @@ -0,0 +1,32 @@ +package org.apache.bigtop.manager.server.command.job.runner.service; + +import jakarta.annotation.Resource; +import lombok.extern.slf4j.Slf4j; +import org.apache.bigtop.manager.common.enums.Command; +import org.apache.bigtop.manager.server.command.CommandIdentifier; +import org.apache.bigtop.manager.server.command.job.runner.AbstractJobRunner; +import org.apache.bigtop.manager.server.enums.CommandLevel; +import org.apache.bigtop.manager.server.service.ServiceService; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; + +@Slf4j +@org.springframework.stereotype.Component +@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) +public class ServiceInstallJobRunner extends AbstractJobRunner { + + @Resource + private ServiceService serviceService; + + @Override + public CommandIdentifier getCommandIdentifier() { + return new CommandIdentifier(CommandLevel.SERVICE, Command.INSTALL); + } + + @Override + public void beforeRun() { + super.beforeRun(); + + serviceService.saveByCommand(getCommandDTO()); + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/service/ServiceRestartJobRunner.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/service/ServiceRestartJobRunner.java new file mode 100644 index 00000000..842a5595 --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/service/ServiceRestartJobRunner.java @@ -0,0 +1,20 @@ +package org.apache.bigtop.manager.server.command.job.runner.service; + +import lombok.extern.slf4j.Slf4j; +import org.apache.bigtop.manager.common.enums.Command; +import org.apache.bigtop.manager.server.command.CommandIdentifier; +import org.apache.bigtop.manager.server.command.job.runner.AbstractJobRunner; +import org.apache.bigtop.manager.server.enums.CommandLevel; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; + +@Slf4j +@org.springframework.stereotype.Component +@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) +public class ServiceRestartJobRunner extends AbstractJobRunner { + + @Override + public CommandIdentifier getCommandIdentifier() { + return new CommandIdentifier(CommandLevel.SERVICE, Command.RESTART); + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/service/ServiceStartJobRunner.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/service/ServiceStartJobRunner.java new file mode 100644 index 00000000..a08095f9 --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/service/ServiceStartJobRunner.java @@ -0,0 +1,20 @@ +package org.apache.bigtop.manager.server.command.job.runner.service; + +import lombok.extern.slf4j.Slf4j; +import org.apache.bigtop.manager.common.enums.Command; +import org.apache.bigtop.manager.server.command.CommandIdentifier; +import org.apache.bigtop.manager.server.command.job.runner.AbstractJobRunner; +import org.apache.bigtop.manager.server.enums.CommandLevel; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; + +@Slf4j +@org.springframework.stereotype.Component +@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) +public class ServiceStartJobRunner extends AbstractJobRunner { + + @Override + public CommandIdentifier getCommandIdentifier() { + return new CommandIdentifier(CommandLevel.SERVICE, Command.START); + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/service/ServiceStopJobRunner.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/service/ServiceStopJobRunner.java new file mode 100644 index 00000000..edc378cc --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/service/ServiceStopJobRunner.java @@ -0,0 +1,20 @@ +package org.apache.bigtop.manager.server.command.job.runner.service; + +import lombok.extern.slf4j.Slf4j; +import org.apache.bigtop.manager.common.enums.Command; +import org.apache.bigtop.manager.server.command.CommandIdentifier; +import org.apache.bigtop.manager.server.command.job.runner.AbstractJobRunner; +import org.apache.bigtop.manager.server.enums.CommandLevel; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; + +@Slf4j +@org.springframework.stereotype.Component +@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) +public class ServiceStopJobRunner extends AbstractJobRunner { + + @Override + public CommandIdentifier getCommandIdentifier() { + return new CommandIdentifier(CommandLevel.SERVICE, Command.STOP); + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/listener/CommandEventListener.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/listener/CommandEventListener.java index 44ce6322..85ee1aec 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/listener/CommandEventListener.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/listener/CommandEventListener.java @@ -18,20 +18,10 @@ package org.apache.bigtop.manager.server.command.listener; import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; -import org.apache.bigtop.manager.common.enums.Command; import org.apache.bigtop.manager.server.command.event.CommandEvent; import org.apache.bigtop.manager.server.command.job.scheduler.JobScheduler; -import org.apache.bigtop.manager.server.command.strategy.AsyncJobStrategy; -import org.apache.bigtop.manager.server.enums.CommandLevel; -import org.apache.bigtop.manager.server.model.dto.CommandDTO; -import org.apache.bigtop.manager.server.orm.entity.Cluster; import org.apache.bigtop.manager.server.orm.entity.Job; -import org.apache.bigtop.manager.server.orm.entity.Stage; -import org.apache.bigtop.manager.server.orm.entity.Task; -import org.apache.bigtop.manager.server.orm.repository.ClusterRepository; import org.apache.bigtop.manager.server.orm.repository.JobRepository; -import org.apache.bigtop.manager.server.orm.repository.StageRepository; -import org.apache.bigtop.manager.server.orm.repository.TaskRepository; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Component; import org.springframework.transaction.event.TransactionalEventListener; @@ -43,59 +33,15 @@ public class CommandEventListener { @Resource private JobRepository jobRepository; - @Resource - private StageRepository stageRepository; - - @Resource - private TaskRepository taskRepository; - - @Resource - private ClusterRepository clusterRepository; - @Resource private JobScheduler jobScheduler; - @Resource - private AsyncJobStrategy asyncJobStrategy; - @Async @TransactionalEventListener public void listen(CommandEvent event) { - log.info("listen CommandEvent: {}", event); - CommandDTO commandDTO = (CommandDTO) event.getSource(); Long jobId = event.getJobId(); Job job = jobRepository.getReferenceById(jobId); jobScheduler.submit(job); -// Boolean failed = asyncJobStrategy.handle(job, JobStrategyType.OVER_ON_FAIL); -// log.info("[CommandEventListener] failed: {}", failed); -// -// if (!failed) { -// // Reload job -// job = jobRepository.getReferenceById(jobId); -// afterJobSuccess(job, commandDTO); -// } - } - - private void afterJobSuccess(Job job, CommandDTO commandDTO) { - CommandLevel commandLevel = commandDTO.getCommandLevel(); - Command command = commandDTO.getCommand(); - if (commandLevel == CommandLevel.CLUSTER && command == Command.CREATE) { - // Link job to cluster after cluster successfully added - Cluster cluster = clusterRepository.findByClusterName(commandDTO.getClusterCommand().getClusterName()).orElse(new Cluster()); - job.setCluster(cluster); - - for (Stage stage : job.getStages()) { - stage.setCluster(cluster); - for (Task task : stage.getTasks()) { - task.setCluster(cluster); - } - - taskRepository.saveAll(stage.getTasks()); - } - - stageRepository.saveAll(job.getStages()); - jobRepository.save(job); - } } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/StageContext.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/StageContext.java index b361b57b..ce516117 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/StageContext.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/StageContext.java @@ -1,11 +1,11 @@ package org.apache.bigtop.manager.server.command.stage.factory; import lombok.Data; -import org.apache.bigtop.manager.common.enums.Command; import org.apache.bigtop.manager.common.utils.JsonUtils; -import org.apache.bigtop.manager.server.enums.CommandLevel; import org.apache.bigtop.manager.server.model.dto.CommandDTO; +import org.apache.bigtop.manager.server.model.dto.ComponentDTO; import org.apache.bigtop.manager.server.model.dto.RepoDTO; +import org.apache.bigtop.manager.server.model.dto.ServiceDTO; import org.apache.bigtop.manager.server.model.dto.command.ClusterCommandDTO; import org.apache.bigtop.manager.server.model.dto.command.HostCommandDTO; @@ -26,9 +26,9 @@ public class StageContext { private List hostnames; - private String serviceName; + private ServiceDTO serviceDTO; - private String componentName; + private ComponentDTO componentDTO; private List repoInfoList; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/StageType.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/StageType.java index df6e6c8b..e4e377c2 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/StageType.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/StageType.java @@ -12,7 +12,7 @@ public enum StageType { COMPONENT_STOP, - COMPONENT_RESTART, + COMPONENT_CHECK, SERVICE_CHECK, diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/component/AbstractComponentStageFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/component/AbstractComponentStageFactory.java new file mode 100644 index 00000000..4cf26f1b --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/component/AbstractComponentStageFactory.java @@ -0,0 +1,141 @@ +package org.apache.bigtop.manager.server.command.stage.factory.component; + +import jakarta.annotation.Resource; +import org.apache.bigtop.manager.common.enums.Command; +import org.apache.bigtop.manager.common.enums.MessageType; +import org.apache.bigtop.manager.common.message.type.CommandPayload; +import org.apache.bigtop.manager.common.message.type.RequestMessage; +import org.apache.bigtop.manager.common.message.type.pojo.CustomCommandInfo; +import org.apache.bigtop.manager.common.message.type.pojo.OSSpecificInfo; +import org.apache.bigtop.manager.common.message.type.pojo.ScriptInfo; +import org.apache.bigtop.manager.common.utils.JsonUtils; +import org.apache.bigtop.manager.server.command.stage.factory.AbstractStageFactory; +import org.apache.bigtop.manager.server.model.dto.*; +import org.apache.bigtop.manager.server.orm.entity.Cluster; +import org.apache.bigtop.manager.server.orm.entity.Task; +import org.apache.bigtop.manager.server.orm.repository.ClusterRepository; +import org.apache.commons.lang3.StringUtils; + +import java.util.ArrayList; +import java.util.List; + +public abstract class AbstractComponentStageFactory extends AbstractStageFactory { + + @Resource + private ClusterRepository clusterRepository; + + protected Cluster cluster; + + @Override + public void doCreateStage() { + cluster = clusterRepository.getReferenceById(context.getClusterId()); + + Command command = getCommand(); + ServiceDTO serviceDTO = context.getServiceDTO(); + ComponentDTO componentDTO = context.getComponentDTO(); + + stage.setName(StringUtils.capitalize(command.toLowerCase()) + " " + componentDTO.getDisplayName()); + stage.setServiceName(serviceDTO.getServiceName()); + stage.setComponentName(componentDTO.getComponentName()); + + List tasks = new ArrayList<>(); + for (String hostname : context.getHostnames()) { + Task task = new Task(); + + // Required fields + task.setName(stage.getName() + " on " + hostname); + task.setHostname(hostname); + task.setCommand(command); + task.setServiceName(serviceDTO.getServiceName()); + task.setStackName(context.getStackName()); + task.setStackVersion(context.getStackVersion()); + + // Context fields + task.setComponentName(componentDTO.getComponentName()); + task.setServiceUser(serviceDTO.getServiceUser()); + task.setServiceGroup(serviceDTO.getServiceGroup()); + task.setCustomCommands(JsonUtils.writeAsString(componentDTO.getCustomCommands())); + task.setCommandScript(JsonUtils.writeAsString(componentDTO.getCommandScript())); + + RequestMessage requestMessage = getMessage(serviceDTO, componentDTO, hostname, command); + task.setContent(JsonUtils.writeAsString(requestMessage)); + task.setMessageId(requestMessage.getMessageId()); + + tasks.add(task); + } + + stage.setTasks(tasks); + } + + protected abstract Command getCommand(); + + private RequestMessage getMessage(ServiceDTO serviceDTO, ComponentDTO componentDTO, String hostname, Command command) { + CommandPayload commandPayload = new CommandPayload(); + commandPayload.setServiceName(serviceDTO.getServiceName()); + commandPayload.setCommand(command); + commandPayload.setServiceUser(serviceDTO.getServiceUser()); + commandPayload.setServiceGroup(serviceDTO.getServiceGroup()); + commandPayload.setStackName(context.getStackName()); + commandPayload.setStackVersion(context.getStackVersion()); + commandPayload.setComponentName(componentDTO.getComponentName()); + commandPayload.setRoot(cluster.getRoot()); + commandPayload.setHostname(hostname); + + commandPayload.setCustomCommands(convertCustomCommandInfo(componentDTO.getCustomCommands())); + commandPayload.setOsSpecifics(convertOSSpecificInfo(serviceDTO.getOsSpecifics())); + commandPayload.setCommandScript(convertScriptInfo(componentDTO.getCommandScript())); + + + RequestMessage requestMessage = new RequestMessage(); + requestMessage.setMessageType(MessageType.COMMAND); + requestMessage.setHostname(hostname); + requestMessage.setMessagePayload(JsonUtils.writeAsString(commandPayload)); + + return requestMessage; + } + + private ScriptInfo convertScriptInfo(ScriptDTO scriptDTO) { + if (scriptDTO == null) { + return null; + } + + ScriptInfo scriptInfo = new ScriptInfo(); + scriptInfo.setScriptId(scriptDTO.getScriptId()); + scriptInfo.setScriptType(scriptDTO.getScriptType()); + scriptInfo.setTimeout(scriptDTO.getTimeout()); + return scriptInfo; + } + + private List convertOSSpecificInfo(List osSpecificDTOs) { + if (osSpecificDTOs == null) { + return new ArrayList<>(); + } + + List osSpecificInfos = new ArrayList<>(); + for (OSSpecificDTO osSpecificDTO : osSpecificDTOs) { + OSSpecificInfo osSpecificInfo = new OSSpecificInfo(); + osSpecificInfo.setOs(osSpecificDTO.getOs()); + osSpecificInfo.setArch(osSpecificDTO.getArch()); + osSpecificInfo.setPackages(osSpecificDTO.getPackages()); + osSpecificInfos.add(osSpecificInfo); + } + + return osSpecificInfos; + } + + private List convertCustomCommandInfo(List customCommandDTOs) { + if (customCommandDTOs == null) { + return new ArrayList<>(); + } + + List customCommandInfos = new ArrayList<>(); + for (CustomCommandDTO customCommandDTO : customCommandDTOs) { + CustomCommandInfo customCommandInfo = new CustomCommandInfo(); + customCommandInfo.setName(customCommandDTO.getName()); + customCommandInfo.setCommandScript(convertScriptInfo(customCommandDTO.getCommandScript())); + customCommandInfos.add(customCommandInfo); + } + + return customCommandInfos; + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/component/ComponentCheckStageFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/component/ComponentCheckStageFactory.java new file mode 100644 index 00000000..1f029e96 --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/component/ComponentCheckStageFactory.java @@ -0,0 +1,23 @@ +package org.apache.bigtop.manager.server.command.stage.factory.component; + +import lombok.extern.slf4j.Slf4j; +import org.apache.bigtop.manager.common.enums.Command; +import org.apache.bigtop.manager.server.command.stage.factory.StageType; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; + +@Slf4j +@org.springframework.stereotype.Component +@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) +public class ComponentCheckStageFactory extends AbstractComponentStageFactory { + + @Override + public StageType getStageType() { + return StageType.COMPONENT_CHECK; + } + + @Override + protected Command getCommand() { + return Command.CHECK; + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/component/ComponentInstallStageFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/component/ComponentInstallStageFactory.java index 0bdcbc22..c577d08d 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/component/ComponentInstallStageFactory.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/component/ComponentInstallStageFactory.java @@ -1,7 +1,7 @@ package org.apache.bigtop.manager.server.command.stage.factory.component; import lombok.extern.slf4j.Slf4j; -import org.apache.bigtop.manager.server.command.stage.factory.AbstractStageFactory; +import org.apache.bigtop.manager.common.enums.Command; import org.apache.bigtop.manager.server.command.stage.factory.StageType; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Scope; @@ -9,7 +9,7 @@ import org.springframework.context.annotation.Scope; @Slf4j @org.springframework.stereotype.Component @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) -public class ComponentInstallStageFactory extends AbstractStageFactory { +public class ComponentInstallStageFactory extends AbstractComponentStageFactory { @Override public StageType getStageType() { @@ -17,6 +17,7 @@ public class ComponentInstallStageFactory extends AbstractStageFactory { } @Override - public void doCreateStage() { + protected Command getCommand() { + return Command.INSTALL; } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/component/ComponentStartStageFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/component/ComponentStartStageFactory.java new file mode 100644 index 00000000..63741b58 --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/component/ComponentStartStageFactory.java @@ -0,0 +1,23 @@ +package org.apache.bigtop.manager.server.command.stage.factory.component; + +import lombok.extern.slf4j.Slf4j; +import org.apache.bigtop.manager.common.enums.Command; +import org.apache.bigtop.manager.server.command.stage.factory.StageType; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; + +@Slf4j +@org.springframework.stereotype.Component +@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) +public class ComponentStartStageFactory extends AbstractComponentStageFactory { + + @Override + public StageType getStageType() { + return StageType.COMPONENT_START; + } + + @Override + protected Command getCommand() { + return Command.START; + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/component/ComponentStopStageFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/component/ComponentStopStageFactory.java new file mode 100644 index 00000000..ae80c64c --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/component/ComponentStopStageFactory.java @@ -0,0 +1,23 @@ +package org.apache.bigtop.manager.server.command.stage.factory.component; + +import lombok.extern.slf4j.Slf4j; +import org.apache.bigtop.manager.common.enums.Command; +import org.apache.bigtop.manager.server.command.stage.factory.StageType; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; + +@Slf4j +@org.springframework.stereotype.Component +@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) +public class ComponentStopStageFactory extends AbstractComponentStageFactory { + + @Override + public StageType getStageType() { + return StageType.COMPONENT_STOP; + } + + @Override + protected Command getCommand() { + return Command.STOP; + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/host/HostCacheStageFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/host/HostCacheStageFactory.java index da061348..8faac1db 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/host/HostCacheStageFactory.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/host/HostCacheStageFactory.java @@ -82,16 +82,21 @@ public class HostCacheStageFactory extends AbstractStageFactory { @Override public void doCreateStage() { + List hostnames = new ArrayList<>(); if (context.getClusterId() == null) { genEmptyCaches(); + hostnames.addAll(context.getHostnames()); } else { genCaches(); + hostnames.addAll(context.getHostnames() == null ? List.of() : context.getHostnames()); + hostnames.addAll(hostRepository.findAllByClusterId(context.getClusterId()).stream().map(Host::getHostname).toList()); } stage.setName(CACHE_STAGE_NAME); List tasks = new ArrayList<>(); - for (String hostname : context.getHostnames()) { + hostnames = hostnames.stream().distinct().toList(); + for (String hostname : hostnames) { Task task = new Task(); task.setName("Cache host for " + hostname); task.setStackName(context.getStackName()); @@ -144,8 +149,7 @@ public class HostCacheStageFactory extends AbstractStageFactory { serviceConfigMap = new HashMap<>(); serviceConfigMappingList.forEach(scm -> { ServiceConfig sc = scm.getServiceConfig(); - List properties = JsonUtils.readFromString(sc.getPropertiesJson(), new TypeReference<>() { - }); + List properties = JsonUtils.readFromString(sc.getPropertiesJson(), new TypeReference<>() {}); String configMapStr = JsonUtils.writeAsString(StackConfigUtils.extractConfigMap(properties)); if (serviceConfigMap.containsKey(sc.getService().getServiceName())) { diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/service/ServiceConfigureStageFactory.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/service/ServiceConfigureStageFactory.java new file mode 100644 index 00000000..96e1c677 --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/factory/service/ServiceConfigureStageFactory.java @@ -0,0 +1,22 @@ +package org.apache.bigtop.manager.server.command.stage.factory.service; + +import lombok.extern.slf4j.Slf4j; +import org.apache.bigtop.manager.server.command.stage.factory.AbstractStageFactory; +import org.apache.bigtop.manager.server.command.stage.factory.StageType; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; + +@Slf4j +@org.springframework.stereotype.Component +@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) +public class ServiceConfigureStageFactory extends AbstractStageFactory { + + @Override + public StageType getStageType() { + return StageType.SERVICE_CONFIGURE; + } + + @Override + public void doCreateStage() { + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/AbstractStageRunner.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/AbstractStageRunner.java index a94cdb9a..7787c892 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/AbstractStageRunner.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/AbstractStageRunner.java @@ -1,25 +1,34 @@ package org.apache.bigtop.manager.server.command.stage.runner; +import com.fasterxml.jackson.core.type.TypeReference; import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.apache.bigtop.manager.common.constants.MessageConstants; +import org.apache.bigtop.manager.common.enums.MessageType; import org.apache.bigtop.manager.common.message.type.BaseCommandMessage; +import org.apache.bigtop.manager.common.message.type.HostCachePayload; import org.apache.bigtop.manager.common.message.type.RequestMessage; import org.apache.bigtop.manager.common.message.type.ResultMessage; +import org.apache.bigtop.manager.common.message.type.pojo.ClusterInfo; +import org.apache.bigtop.manager.common.message.type.pojo.ComponentInfo; +import org.apache.bigtop.manager.common.message.type.pojo.RepoInfo; import org.apache.bigtop.manager.common.utils.JsonUtils; +import org.apache.bigtop.manager.server.enums.CommandLevel; import org.apache.bigtop.manager.server.enums.JobState; import org.apache.bigtop.manager.server.holder.SpringContextHolder; -import org.apache.bigtop.manager.server.orm.entity.Stage; -import org.apache.bigtop.manager.server.orm.entity.Task; -import org.apache.bigtop.manager.server.orm.repository.StageRepository; -import org.apache.bigtop.manager.server.orm.repository.TaskRepository; +import org.apache.bigtop.manager.server.model.dto.CommandDTO; +import org.apache.bigtop.manager.server.model.dto.PropertyDTO; +import org.apache.bigtop.manager.server.model.mapper.RepoMapper; +import org.apache.bigtop.manager.server.orm.entity.*; +import org.apache.bigtop.manager.server.orm.repository.*; +import org.apache.bigtop.manager.server.utils.StackConfigUtils; -import java.util.ArrayList; -import java.util.List; +import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; -import static org.apache.bigtop.manager.common.constants.Constants.COMMAND_MESSAGE_RESPONSE_TIMEOUT; +import static org.apache.bigtop.manager.common.constants.Constants.*; @Slf4j public abstract class AbstractStageRunner implements StageRunner { @@ -30,6 +39,27 @@ public abstract class AbstractStageRunner implements StageRunner { @Resource private TaskRepository taskRepository; + @Resource + private HostComponentRepository hostComponentRepository; + + @Resource + private ServiceRepository serviceRepository; + + @Resource + private ServiceConfigMappingRepository serviceConfigMappingRepository; + + @Resource + private RepoRepository repoRepository; + + @Resource + private SettingRepository settingRepository; + + @Resource + private HostRepository hostRepository; + + @Resource + private ComponentRepository componentRepository; + protected Stage stage; @Override @@ -43,10 +73,17 @@ public abstract class AbstractStageRunner implements StageRunner { List> futures = new ArrayList<>(); for (Task task : stage.getTasks()) { - task.setState(JobState.PROCESSING); - taskRepository.save(task); + beforeRunTask(task); + + // TODO temp code, need to remove this + String content = task.getContent(); + CommandDTO commandDTO = JsonUtils.readFromString(task.getJob().getPayload(), CommandDTO.class); + if (commandDTO.getCommandLevel() != CommandLevel.CLUSTER && stage.getName().equals(CACHE_STAGE_NAME)) { + content = generatePayload(task); + } + // End temp code - BaseCommandMessage message = JsonUtils.readFromString(task.getContent(), RequestMessage.class); + BaseCommandMessage message = JsonUtils.readFromString(content, RequestMessage.class); message.setTaskId(task.getId()); message.setStageId(stage.getId()); message.setJobId(stage.getJob().getId()); @@ -57,8 +94,11 @@ public abstract class AbstractStageRunner implements StageRunner { log.info("Execute task {} completed: {}", task.getId(), res); boolean taskSuccess = res.getCode() == MessageConstants.SUCCESS_CODE; - task.setState(taskSuccess ? JobState.SUCCESSFUL : JobState.FAILED); - taskRepository.save(task); + if (taskSuccess) { + onTaskSuccess(task); + } else { + onTaskFailure(task); + } return taskSuccess; })); @@ -98,4 +138,136 @@ public abstract class AbstractStageRunner implements StageRunner { stage.setState(JobState.FAILED); stageRepository.save(stage); } + + @Override + public void beforeRunTask(Task task) { + task.setState(JobState.PROCESSING); + taskRepository.save(task); + } + + @Override + public void onTaskSuccess(Task task) { + task.setState(JobState.SUCCESSFUL); + taskRepository.save(task); + } + + @Override + public void onTaskFailure(Task task) { + task.setState(JobState.FAILED); + taskRepository.save(task); + } + + public String generatePayload(Task task) { + Cluster cluster = task.getCluster(); + RequestMessage requestMessage = getMessage(cluster, task.getHostname()); + log.info("[generatePayload]-[HostCacheJobFactory-requestMessage]: {}", requestMessage); + return JsonUtils.writeAsString(requestMessage); + } + + public RequestMessage getMessage(Cluster cluster, String hostname) { + ClusterInfo clusterInfo = new ClusterInfo(); + Map componentInfoMap = new HashMap<>(); + Map> serviceConfigMap = new HashMap<>(); + Map> hostMap = new HashMap<>(); + List repoList = new ArrayList<>(); + Map> userMap = new HashMap<>(); + Map settingsMap = new HashMap<>(); + + + Long clusterId = cluster.getId(); + + String clusterName = cluster.getClusterName(); + String stackName = cluster.getStack().getStackName(); + String stackVersion = cluster.getStack().getStackVersion(); + + List services = serviceRepository.findAllByClusterId(clusterId); + List serviceConfigMappingList = serviceConfigMappingRepository.findAllGroupLastest(clusterId); + List hostComponents = hostComponentRepository.findAllByComponentClusterId(clusterId); + List repos = repoRepository.findAllByCluster(cluster); + Iterable settings = settingRepository.findAll(); + List hostList = hostRepository.findAllByClusterId(clusterId); + + //Wrapper clusterInfo for HostCacheMessage + clusterInfo.setClusterName(clusterName); + clusterInfo.setStackName(stackName); + clusterInfo.setStackVersion(stackVersion); + clusterInfo.setUserGroup(cluster.getUserGroup()); + clusterInfo.setRepoTemplate(cluster.getRepoTemplate()); + clusterInfo.setRoot(cluster.getRoot()); + clusterInfo.setPackages(Arrays.asList(cluster.getPackages().split(","))); + + //Wrapper serviceConfigMap for HostCacheMessage + serviceConfigMappingList.forEach(scm -> { + ServiceConfig sc = scm.getServiceConfig(); + List properties = JsonUtils.readFromString(sc.getPropertiesJson(), new TypeReference<>() { + }); + String configMapStr = JsonUtils.writeAsString(StackConfigUtils.extractConfigMap(properties)); + + if (serviceConfigMap.containsKey(sc.getService().getServiceName())) { + serviceConfigMap.get(sc.getService().getServiceName()).put(sc.getTypeName(), configMapStr); + } else { + Map hashMap = new HashMap<>(); + hashMap.put(sc.getTypeName(), configMapStr); + serviceConfigMap.put(sc.getService().getServiceName(), hashMap); + } + }); + + //Wrapper hostMap for HostCacheMessage + hostComponents.forEach(x -> { + if (hostMap.containsKey(x.getComponent().getComponentName())) { + hostMap.get(x.getComponent().getComponentName()).add(x.getHost().getHostname()); + } else { + Set set = new HashSet<>(); + set.add(x.getHost().getHostname()); + hostMap.put(x.getComponent().getComponentName(), set); + } + hostMap.get(x.getComponent().getComponentName()).add(x.getHost().getHostname()); + }); + + Set hostNameSet = hostList.stream().map(Host::getHostname).collect(Collectors.toSet()); + hostMap.put(ALL_HOST_KEY, hostNameSet); + + //Wrapper repoList for HostCacheMessage + repos.forEach(repo -> { + RepoInfo repoInfo = RepoMapper.INSTANCE.fromEntity2Message(repo); + repoList.add(repoInfo); + }); + + //Wrapper userMap for HostCacheMessage + services.forEach(x -> userMap.put(x.getServiceUser(), Set.of(x.getServiceGroup()))); + + //Wrapper settings for HostCacheMessage + settings.forEach(x -> settingsMap.put(x.getTypeName(), x.getConfigData())); + + //Wrapper componentInfoList for HostCacheMessage + List componentList = componentRepository.findAll(); + componentList.forEach(c -> { + ComponentInfo componentInfo = new ComponentInfo(); + componentInfo.setComponentName(c.getComponentName()); + componentInfo.setCommandScript(c.getCommandScript()); + componentInfo.setDisplayName(c.getDisplayName()); + componentInfo.setCategory(c.getCategory()); + componentInfo.setCustomCommands(c.getCustomCommands()); + componentInfo.setServiceName(c.getService().getServiceName()); + componentInfoMap.put(c.getComponentName(), componentInfo); + }); + + HostCachePayload hostCachePayload = new HostCachePayload(); + hostCachePayload.setHostname(hostname); + + hostCachePayload.setClusterInfo(clusterInfo); + hostCachePayload.setConfigurations(serviceConfigMap); + hostCachePayload.setClusterHostInfo(hostMap); + hostCachePayload.setRepoInfo(repoList); + hostCachePayload.setSettings(settingsMap); + hostCachePayload.setUserInfo(userMap); + hostCachePayload.setComponentInfo(componentInfoMap); + + RequestMessage requestMessage = new RequestMessage(); + requestMessage.setMessageType(MessageType.HOST_CACHE); + requestMessage.setHostname(hostname); + requestMessage.setMessagePayload(JsonUtils.writeAsString(hostCachePayload)); + + return requestMessage; + } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/StageRunner.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/StageRunner.java index 75c98093..c82a43a7 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/StageRunner.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/StageRunner.java @@ -2,6 +2,7 @@ package org.apache.bigtop.manager.server.command.stage.runner; import org.apache.bigtop.manager.server.command.stage.factory.StageType; import org.apache.bigtop.manager.server.orm.entity.Stage; +import org.apache.bigtop.manager.server.orm.entity.Task; public interface StageRunner { @@ -16,4 +17,10 @@ public interface StageRunner { void onSuccess(); void onFailure(); + + void beforeRunTask(Task task); + + void onTaskSuccess(Task task); + + void onTaskFailure(Task task); } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/StageRunners.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/StageRunners.java index 201da569..082388a3 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/StageRunners.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/StageRunners.java @@ -23,7 +23,7 @@ public class StageRunners { load(); } - StageContext context = JsonUtils.readFromString(stage.getContext()); + StageContext context = JsonUtils.readFromString(stage.getContext(), StageContext.class); StageType stageType = context.getStageType(); String beanName = STAGE_RUNNERS.get(stageType); diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/component/ComponentCheckStageRunner.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/component/ComponentCheckStageRunner.java new file mode 100644 index 00000000..c4a9afb4 --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/component/ComponentCheckStageRunner.java @@ -0,0 +1,20 @@ +package org.apache.bigtop.manager.server.command.stage.runner.component; + +import lombok.extern.slf4j.Slf4j; +import org.apache.bigtop.manager.common.enums.Command; +import org.apache.bigtop.manager.server.command.stage.factory.StageType; +import org.apache.bigtop.manager.server.command.stage.factory.component.AbstractComponentStageFactory; +import org.apache.bigtop.manager.server.command.stage.runner.AbstractStageRunner; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; + +@Slf4j +@org.springframework.stereotype.Component +@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) +public class ComponentCheckStageRunner extends AbstractStageRunner { + + @Override + public StageType getStageType() { + return StageType.COMPONENT_CHECK; + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/component/ComponentInstallStageRunner.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/component/ComponentInstallStageRunner.java new file mode 100644 index 00000000..434e4b98 --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/component/ComponentInstallStageRunner.java @@ -0,0 +1,38 @@ +package org.apache.bigtop.manager.server.command.stage.runner.component; + +import jakarta.annotation.Resource; +import lombok.extern.slf4j.Slf4j; +import org.apache.bigtop.manager.server.command.stage.factory.StageType; +import org.apache.bigtop.manager.server.command.stage.runner.AbstractStageRunner; +import org.apache.bigtop.manager.server.enums.MaintainState; +import org.apache.bigtop.manager.server.orm.entity.HostComponent; +import org.apache.bigtop.manager.server.orm.entity.Task; +import org.apache.bigtop.manager.server.orm.repository.HostComponentRepository; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; + +@Slf4j +@org.springframework.stereotype.Component +@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) +public class ComponentInstallStageRunner extends AbstractStageRunner { + + @Resource + private HostComponentRepository hostComponentRepository; + + @Override + public StageType getStageType() { + return StageType.COMPONENT_INSTALL; + } + + @Override + public void onTaskSuccess(Task task) { + super.onTaskSuccess(task); + + Long clusterId = task.getCluster().getId(); + String componentName = task.getComponentName(); + String hostname = task.getHostname(); + HostComponent hostComponent = hostComponentRepository.findByComponentClusterIdAndComponentComponentNameAndHostHostname(clusterId, componentName, hostname); + hostComponent.setState(MaintainState.INSTALLED); + hostComponentRepository.save(hostComponent); + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/component/ComponentStartStageRunner.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/component/ComponentStartStageRunner.java new file mode 100644 index 00000000..bb82d963 --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/component/ComponentStartStageRunner.java @@ -0,0 +1,43 @@ +package org.apache.bigtop.manager.server.command.stage.runner.component; + +import jakarta.annotation.Resource; +import lombok.extern.slf4j.Slf4j; +import org.apache.bigtop.manager.server.command.stage.factory.StageType; +import org.apache.bigtop.manager.server.command.stage.runner.AbstractStageRunner; +import org.apache.bigtop.manager.server.enums.MaintainState; +import org.apache.bigtop.manager.server.orm.entity.HostComponent; +import org.apache.bigtop.manager.server.orm.entity.Task; +import org.apache.bigtop.manager.server.orm.repository.HostComponentRepository; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; + +@Slf4j +@org.springframework.stereotype.Component +@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) +public class ComponentStartStageRunner extends AbstractStageRunner { + + @Resource + private HostComponentRepository hostComponentRepository; + + @Override + public StageType getStageType() { + return StageType.COMPONENT_START; + } + + @Override + public void onTaskSuccess(Task task) { + super.onTaskSuccess(task); + + Long clusterId = task.getCluster().getId(); + String componentName = task.getComponentName(); + String hostname = task.getHostname(); + HostComponent hostComponent = hostComponentRepository.findByComponentClusterIdAndComponentComponentNameAndHostHostname(clusterId, componentName, hostname); + hostComponent.setState(MaintainState.STARTED); + hostComponentRepository.save(hostComponent); + } + + @Override + public void onSuccess() { + super.onSuccess(); + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/component/ComponentStopStageRunner.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/component/ComponentStopStageRunner.java new file mode 100644 index 00000000..4458d190 --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/component/ComponentStopStageRunner.java @@ -0,0 +1,39 @@ +package org.apache.bigtop.manager.server.command.stage.runner.component; + +import jakarta.annotation.Resource; +import lombok.extern.slf4j.Slf4j; +import org.apache.bigtop.manager.server.command.stage.factory.StageType; +import org.apache.bigtop.manager.server.command.stage.runner.AbstractStageRunner; +import org.apache.bigtop.manager.server.enums.MaintainState; +import org.apache.bigtop.manager.server.orm.entity.HostComponent; +import org.apache.bigtop.manager.server.orm.entity.Task; +import org.apache.bigtop.manager.server.orm.repository.HostComponentRepository; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; + +@Slf4j +@org.springframework.stereotype.Component +@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) +public class ComponentStopStageRunner extends AbstractStageRunner { + + @Resource + private HostComponentRepository hostComponentRepository; + + + @Override + public StageType getStageType() { + return StageType.COMPONENT_STOP; + } + + @Override + public void onTaskSuccess(Task task) { + super.onTaskSuccess(task); + + Long clusterId = task.getCluster().getId(); + String componentName = task.getComponentName(); + String hostname = task.getHostname(); + HostComponent hostComponent = hostComponentRepository.findByComponentClusterIdAndComponentComponentNameAndHostHostname(clusterId, componentName, hostname); + hostComponent.setState(MaintainState.STOPPED); + hostComponentRepository.save(hostComponent); + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/service/ServiceConfigureStageRunner.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/service/ServiceConfigureStageRunner.java new file mode 100644 index 00000000..8c6c0110 --- /dev/null +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/runner/service/ServiceConfigureStageRunner.java @@ -0,0 +1,18 @@ +package org.apache.bigtop.manager.server.command.stage.runner.service; + +import lombok.extern.slf4j.Slf4j; +import org.apache.bigtop.manager.server.command.stage.factory.StageType; +import org.apache.bigtop.manager.server.command.stage.runner.AbstractStageRunner; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; + +@Slf4j +@org.springframework.stereotype.Component +@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) +public class ServiceConfigureStageRunner extends AbstractStageRunner { + + @Override + public StageType getStageType() { + return StageType.SERVICE_CONFIGURE; + } +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/strategy/AbstractJobStrategy.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/strategy/AbstractJobStrategy.java deleted file mode 100644 index 2ea6f99c..00000000 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/strategy/AbstractJobStrategy.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.apache.bigtop.manager.server.command.strategy; - -import jakarta.annotation.Resource; -import org.apache.bigtop.manager.server.exception.ServerException; -import org.apache.bigtop.manager.server.orm.entity.Stage; -import org.springframework.context.ApplicationContext; - -public abstract class AbstractJobStrategy implements JobStrategy { - @Resource - private ApplicationContext applicationContext; - - public StageCallback getStageCallback(Stage stage) { - String callbackClassName = stage.getCallbackClassName(); - if (callbackClassName != null) { - try { - Class clazz = Class.forName(callbackClassName); - if (StageCallback.class.isAssignableFrom(clazz)) { - return (StageCallback) applicationContext.getBean(clazz); - } - } catch (Exception e) { - throw new ServerException(e); - } - } - return null; - } -} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/strategy/AsyncJobStrategy.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/strategy/AsyncJobStrategy.java deleted file mode 100644 index 789d2d78..00000000 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/strategy/AsyncJobStrategy.java +++ /dev/null @@ -1,184 +0,0 @@ -package org.apache.bigtop.manager.server.command.strategy; - -import jakarta.annotation.Resource; -import lombok.extern.slf4j.Slf4j; -import org.apache.bigtop.manager.common.constants.MessageConstants; -import org.apache.bigtop.manager.common.message.type.BaseCommandMessage; -import org.apache.bigtop.manager.common.message.type.RequestMessage; -import org.apache.bigtop.manager.common.message.type.ResultMessage; -import org.apache.bigtop.manager.common.utils.JsonUtils; -import org.apache.bigtop.manager.server.enums.JobState; -import org.apache.bigtop.manager.server.enums.JobStrategyType; -import org.apache.bigtop.manager.server.holder.SpringContextHolder; -import org.apache.bigtop.manager.server.orm.entity.CommandLog; -import org.apache.bigtop.manager.server.orm.entity.Job; -import org.apache.bigtop.manager.server.orm.entity.Stage; -import org.apache.bigtop.manager.server.orm.entity.Task; -import org.apache.bigtop.manager.server.orm.repository.CommandLogRepository; -import org.apache.bigtop.manager.server.orm.repository.JobRepository; -import org.apache.bigtop.manager.server.orm.repository.StageRepository; -import org.apache.bigtop.manager.server.orm.repository.TaskRepository; -import org.springframework.stereotype.Component; - -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - -import static org.apache.bigtop.manager.common.constants.Constants.CACHE_STAGE_NAME; -import static org.apache.bigtop.manager.common.constants.Constants.COMMAND_MESSAGE_RESPONSE_TIMEOUT; - -@Slf4j -@Component -public class AsyncJobStrategy extends AbstractJobStrategy { - - @Resource - private JobRepository jobRepository; - - @Resource - private StageRepository stageRepository; - - @Resource - private TaskRepository taskRepository; - - @Resource - private CommandLogRepository commandLogRepository; - - private CountDownLatch countDownLatch; - - private AtomicBoolean failed; - - @Override - public Boolean handle(Job job, JobStrategyType strategyType) { - failed = new AtomicBoolean(false); - List stageCompletedList = new ArrayList<>(); - - // Job state change to processing - job.setState(JobState.PROCESSING); - jobRepository.save(job); - - // Sort stage - List stages = job.getStages(); - stages.sort(Comparator.comparingInt(Stage::getStageOrder)); - - LinkedBlockingQueue pipeLineQueue = new LinkedBlockingQueue<>(stages); - while (!pipeLineQueue.isEmpty()) { - Stage stage = pipeLineQueue.poll(); - - // Stage state change to processing - stage.setState(JobState.PROCESSING); - stageRepository.save(stage); - StageCallback stageCallback = getStageCallback(stage); - - if (stageCallback != null) { - stageCallback.beforeStage(stage); - } - - countDownLatch = new CountDownLatch(stage.getTasks().size()); - for (Task task : stage.getTasks()) { - // Task state change to processing - task.setState(JobState.PROCESSING); - taskRepository.save(task); - - String content = task.getContent(); - if (stageCallback != null && stage.getName().equals(CACHE_STAGE_NAME)) { - content = stageCallback.generatePayload(task); - } - - BaseCommandMessage message = JsonUtils.readFromString(content, RequestMessage.class); - message.setTaskId(task.getId()); - message.setStageId(stage.getId()); - message.setJobId(job.getId()); - log.info("[AsyncJobStrategy] [BaseCommandMessage]: {}", message); - - SpringContextHolder.getServerWebSocket().sendMessage(task.getHostname(), message, this::call); - } - - boolean timeoutFlag = false; - try { - timeoutFlag = countDownLatch.await(COMMAND_MESSAGE_RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS); - } catch (Exception e) { - stage.setState(JobState.FAILED); - } - - if (timeoutFlag && !failed.get()) { - stage.setState(JobState.SUCCESSFUL); - } else { - log.error("stage failed or timeout, cancel remain stages, [timeoutFlag: {}], [failed: {}]", timeoutFlag, failed); - stage.setState(JobState.FAILED); - if (strategyType == JobStrategyType.OVER_ON_FAIL) { - releaseRemainStages(pipeLineQueue); - } - } - stageCompletedList.add(stage); - stageRepository.save(stage); - - if (stageCallback != null) { - stageCallback.afterStage(stage); - } - } - - boolean stageSuccessful = stageCompletedList.stream().allMatch(stage -> stage.getState() == JobState.SUCCESSFUL); - if (stageSuccessful) { - job.setState(JobState.SUCCESSFUL); - } else { - job.setState(JobState.FAILED); - } - jobRepository.save(job); - return failed.get(); - } - - - /** - * callback - */ - public void call(ResultMessage resultMessage) { - log.info("Execute RequestMessage completed, {}", resultMessage); - Task task = taskRepository.getReferenceById(resultMessage.getTaskId()); - - if (resultMessage.getCode() == MessageConstants.SUCCESS_CODE) { - task.setState(JobState.SUCCESSFUL); - } else { - task.setState(JobState.FAILED); - failed.set(true); - } - taskRepository.save(task); - saveCommandLog(resultMessage.getResult(), task); - - countDownLatch.countDown(); - } - - - /** - * Release remaining Stages - * Execute when failed or timeout - */ - private void releaseRemainStages(LinkedBlockingQueue pipeLineQueue) { - if (!pipeLineQueue.isEmpty()) { - List remainStages = new ArrayList<>(pipeLineQueue.size()); - pipeLineQueue.drainTo(remainStages); - for (Stage stage : remainStages) { - // Setting the status of the remaining stages to CANCELED - stage.setState(JobState.CANCELED); - stageRepository.save(stage); - - stage.getTasks().forEach(t -> t.setState(JobState.CANCELED)); - taskRepository.saveAll(stage.getTasks()); - } - } - } - - private void saveCommandLog(String result, Task task) { - CommandLog commandLog = new CommandLog(); - commandLog.setTask(task); - commandLog.setStage(task.getStage()); - commandLog.setJob(task.getJob()); - commandLog.setHostname(task.getHostname()); - commandLog.setResult(result); - commandLogRepository.save(commandLog); - } - -} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/strategy/JobStrategy.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/strategy/JobStrategy.java deleted file mode 100644 index 8807d726..00000000 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/strategy/JobStrategy.java +++ /dev/null @@ -1,8 +0,0 @@ -package org.apache.bigtop.manager.server.command.strategy; - -import org.apache.bigtop.manager.server.enums.JobStrategyType; -import org.apache.bigtop.manager.server.orm.entity.Job; - -public interface JobStrategy { - Boolean handle(Job job, JobStrategyType strategyType); -} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/strategy/StageCallback.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/strategy/StageCallback.java deleted file mode 100644 index 95b17bcf..00000000 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/strategy/StageCallback.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.apache.bigtop.manager.server.command.strategy; - -import org.apache.bigtop.manager.server.orm.entity.Stage; -import org.apache.bigtop.manager.server.orm.entity.Task; - -public interface StageCallback { - - default void beforeStage(Stage stage) { - } - - default void afterStage(Stage stage) { - } - - default String generatePayload(Task task) { - return task.getContent(); - } -} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/strategy/SyncJobStrategy.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/strategy/SyncJobStrategy.java deleted file mode 100644 index 5686038a..00000000 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/strategy/SyncJobStrategy.java +++ /dev/null @@ -1,109 +0,0 @@ -package org.apache.bigtop.manager.server.command.strategy; - -import jakarta.annotation.Resource; -import lombok.extern.slf4j.Slf4j; -import org.apache.bigtop.manager.common.constants.MessageConstants; -import org.apache.bigtop.manager.common.message.type.BaseCommandMessage; -import org.apache.bigtop.manager.common.message.type.RequestMessage; -import org.apache.bigtop.manager.common.message.type.ResultMessage; -import org.apache.bigtop.manager.common.utils.JsonUtils; -import org.apache.bigtop.manager.server.enums.JobState; -import org.apache.bigtop.manager.server.enums.JobStrategyType; -import org.apache.bigtop.manager.server.holder.SpringContextHolder; -import org.apache.bigtop.manager.server.orm.entity.Job; -import org.apache.bigtop.manager.server.orm.entity.Stage; -import org.apache.bigtop.manager.server.orm.entity.Task; -import org.apache.bigtop.manager.server.orm.repository.JobRepository; -import org.apache.bigtop.manager.server.orm.repository.StageRepository; -import org.apache.bigtop.manager.server.orm.repository.TaskRepository; -import org.springframework.stereotype.Component; - -import java.util.Comparator; -import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; - -import static org.apache.bigtop.manager.common.constants.Constants.CACHE_STAGE_NAME; - -@Slf4j -@Component -public class SyncJobStrategy extends AbstractJobStrategy { - - @Resource - private JobRepository jobRepository; - - @Resource - private StageRepository stageRepository; - - @Resource - private TaskRepository taskRepository; - - @Override - public Boolean handle(Job job, JobStrategyType strategyType) { - AtomicBoolean failed = new AtomicBoolean(false); - List stages = job.getStages(); - - // Sort stage - stages.sort(Comparator.comparingInt(Stage::getStageOrder)); - - job.setState(JobState.PROCESSING); - jobRepository.save(job); - for (Stage stage : stages) { - stage.setState(JobState.PROCESSING); - stageRepository.save(stage); - - StageCallback stageCallback = getStageCallback(stage); - if (stageCallback != null) { - stageCallback.beforeStage(stage); - } - - List tasks = stage.getTasks(); - for (Task task : tasks) { - task.setState(JobState.PROCESSING); - taskRepository.save(task); - - String content = task.getContent(); - if (stageCallback != null && stage.getName().equals(CACHE_STAGE_NAME)) { - content = stageCallback.generatePayload(task); - } - BaseCommandMessage message = JsonUtils.readFromString(content, RequestMessage.class); - log.info("[SyncJobStrategy] [BaseCommandMessage]: {}", message); - String hostname = task.getHostname(); - ResultMessage res = SpringContextHolder.getServerWebSocket().sendMessage(hostname, message); - if (res == null || res.getCode() != MessageConstants.SUCCESS_CODE) { - task.setState(JobState.FAILED); - failed.set(true); - } else { - task.setState(JobState.SUCCESSFUL); - } - taskRepository.save(task); - - if (failed.get() && strategyType == JobStrategyType.OVER_ON_FAIL) { - break; - } - } - - if (failed.get()) { - stage.setState(JobState.FAILED); - } else { - stage.setState(JobState.SUCCESSFUL); - } - stageRepository.save(stage); - - if (stageCallback != null) { - stageCallback.afterStage(stage); - } - - if (failed.get() && strategyType == JobStrategyType.OVER_ON_FAIL) { - break; - } - } - if (failed.get()) { - job.setState(JobState.FAILED); - } else { - job.setState(JobState.SUCCESSFUL); - } - jobRepository.save(job); - - return failed.get(); - } -} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/mapper/ClusterMapper.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/mapper/ClusterMapper.java index 039c51d1..d4141e2d 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/mapper/ClusterMapper.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/mapper/ClusterMapper.java @@ -2,6 +2,7 @@ package org.apache.bigtop.manager.server.model.mapper; import org.apache.bigtop.manager.server.model.dto.ClusterDTO; import org.apache.bigtop.manager.server.model.dto.StackDTO; +import org.apache.bigtop.manager.server.model.dto.command.ClusterCommandDTO; import org.apache.bigtop.manager.server.model.req.ClusterReq; import org.apache.bigtop.manager.server.model.vo.ClusterVO; import org.apache.bigtop.manager.server.orm.entity.Cluster; @@ -18,6 +19,8 @@ public interface ClusterMapper { ClusterDTO fromReq2DTO(ClusterReq clusterReq); + ClusterDTO fromCommand2DTO(ClusterCommandDTO clusterCommandDTO); + Cluster fromDTO2Entity(ClusterDTO clusterDTO); @Mapping(target = "stackName", source = "cluster.stack.stackName") diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/orm/entity/Service.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/orm/entity/Service.java index 38902f58..3f73e81d 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/orm/entity/Service.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/orm/entity/Service.java @@ -3,9 +3,6 @@ package org.apache.bigtop.manager.server.orm.entity; import jakarta.persistence.*; import lombok.Data; import lombok.EqualsAndHashCode; -import org.apache.bigtop.manager.server.enums.MaintainState; - -import java.util.List; @Data @EqualsAndHashCode(callSuper = true) @@ -44,10 +41,6 @@ public class Service extends BaseEntity { @Column(name = "required_service") private String requiredServices; - @Enumerated(EnumType.STRING) - @Column(name = "state") - private MaintainState state; - @ManyToOne @JoinColumn(name = "cluster_id", foreignKey = @ForeignKey(ConstraintMode.NO_CONSTRAINT)) private Cluster cluster; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/orm/entity/Stage.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/orm/entity/Stage.java index dc5cac81..e5841fbf 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/orm/entity/Stage.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/orm/entity/Stage.java @@ -37,11 +37,6 @@ public class Stage extends BaseEntity { @Column(name = "component_name") private String componentName; - @Lob - @Basic(fetch = FetchType.LAZY) - @Column(name = "payload", length = 16777216) - private String payload; - @Lob @Basic(fetch = FetchType.LAZY) @Column(name = "context", length = 16777216) @@ -58,8 +53,4 @@ public class Stage extends BaseEntity { @ToString.Exclude @OneToMany(mappedBy = "stage") private List tasks; - - @Column(name = "callback_class_name") - private String callbackClassName; - } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/orm/repository/HostComponentRepository.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/orm/repository/HostComponentRepository.java index ffdbea1e..21aa3856 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/orm/repository/HostComponentRepository.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/orm/repository/HostComponentRepository.java @@ -10,6 +10,8 @@ public interface HostComponentRepository extends JpaRepository findAllByComponentClusterIdAndComponentComponentName(Long clusterId, String componentName); + HostComponent findByComponentClusterIdAndComponentComponentNameAndHostHostname(Long clusterId, String componentName, String hostnames); + List findAllByComponentClusterIdAndComponentComponentNameAndHostHostnameIn(Long clusterId, String componentName, List hostnames); List findAllByComponentClusterId(Long clusterId); diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/CommandServiceImpl.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/CommandServiceImpl.java index f8033378..6c552f25 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/CommandServiceImpl.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/CommandServiceImpl.java @@ -1,14 +1,14 @@ package org.apache.bigtop.manager.server.service.impl; -import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.apache.bigtop.manager.server.command.CommandIdentifier; +import org.apache.bigtop.manager.server.command.event.CommandEvent; import org.apache.bigtop.manager.server.command.job.factory.JobContext; import org.apache.bigtop.manager.server.command.job.factory.JobFactories; import org.apache.bigtop.manager.server.command.job.factory.JobFactory; -import org.apache.bigtop.manager.server.command.job.scheduler.JobScheduler; import org.apache.bigtop.manager.server.command.job.validator.ValidatorContext; import org.apache.bigtop.manager.server.command.job.validator.ValidatorExecutionChain; +import org.apache.bigtop.manager.server.holder.SpringContextHolder; import org.apache.bigtop.manager.server.model.dto.CommandDTO; import org.apache.bigtop.manager.server.model.mapper.JobMapper; import org.apache.bigtop.manager.server.model.vo.CommandVO; @@ -20,9 +20,6 @@ import org.springframework.transaction.annotation.Transactional; @org.springframework.stereotype.Service public class CommandServiceImpl implements CommandService { - @Resource - private JobScheduler jobScheduler; - @Override @Transactional public CommandVO command(CommandDTO commandDTO) { @@ -39,13 +36,10 @@ public class CommandServiceImpl implements CommandService { JobFactory jobFactory = JobFactories.getJobFactory(commandIdentifier); Job job = jobFactory.createJob(jobContext); - // Submit job to scheduler - jobScheduler.submit(job); - // Publish command event -// CommandEvent event = new CommandEvent(commandDTO); -// event.setJobId(job.getId()); -// SpringContextHolder.getApplicationContext().publishEvent(event); + CommandEvent event = new CommandEvent(commandDTO); + event.setJobId(job.getId()); + SpringContextHolder.getApplicationContext().publishEvent(event); return JobMapper.INSTANCE.fromEntity2CommandVO(job); } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/ServiceServiceImpl.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/ServiceServiceImpl.java index a43a0a8d..8f3c49e4 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/ServiceServiceImpl.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/ServiceServiceImpl.java @@ -2,9 +2,7 @@ package org.apache.bigtop.manager.server.service.impl; import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; -import org.apache.bigtop.manager.server.enums.ApiExceptionEnum; import org.apache.bigtop.manager.server.enums.MaintainState; -import org.apache.bigtop.manager.server.exception.ApiException; import org.apache.bigtop.manager.server.model.dto.*; import org.apache.bigtop.manager.server.model.dto.command.ServiceCommandDTO; import org.apache.bigtop.manager.server.model.mapper.ComponentMapper; @@ -15,7 +13,6 @@ import org.apache.bigtop.manager.server.orm.repository.*; import org.apache.bigtop.manager.server.service.ConfigService; import org.apache.bigtop.manager.server.service.ServiceService; import org.apache.bigtop.manager.server.utils.StackUtils; -import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.tuple.ImmutablePair; import java.util.List; @@ -88,7 +85,6 @@ public class ServiceServiceImpl implements ServiceService { ServiceDTO serviceDTO = serviceNameToDTO.get(serviceName); Service service = ServiceMapper.INSTANCE.fromDTO2Entity(serviceDTO, cluster); - service.setState(MaintainState.INSTALLED); service = serviceRepository.save(service); // Init config for new installed service @@ -117,7 +113,7 @@ public class ServiceServiceImpl implements ServiceService { hostComponentOptional.ifPresent(value -> hostComponent.setId(value.getId())); hostComponent.setHost(host); hostComponent.setComponent(component); - hostComponent.setState(MaintainState.INSTALLED); + hostComponent.setState(MaintainState.UNINSTALLED); hostComponentRepository.save(hostComponent); } } diff --git a/bigtop-manager-stack/bigtop-manager-stack-common/src/main/java/org/apache/bigtop/manager/stack/common/utils/LocalSettings.java b/bigtop-manager-stack/bigtop-manager-stack-common/src/main/java/org/apache/bigtop/manager/stack/common/utils/LocalSettings.java index 0c2a0221..aaa8930d 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-common/src/main/java/org/apache/bigtop/manager/stack/common/utils/LocalSettings.java +++ b/bigtop-manager-stack/bigtop-manager-stack-common/src/main/java/org/apache/bigtop/manager/stack/common/utils/LocalSettings.java @@ -80,9 +80,9 @@ public class LocalSettings { return userMap; } - public static Set packages() { + public static List packages() { ClusterInfo cluster = cluster(); - return Optional.ofNullable(cluster.getPackages()).orElse(Set.of()); + return Optional.ofNullable(cluster.getPackages()).orElse(List.of()); } public static List repos() { diff --git a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/hooks/InstallHookImpl.java b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/hooks/InstallHookImpl.java index 853a1b5a..1ed48e8c 100644 --- a/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/hooks/InstallHookImpl.java +++ b/bigtop-manager-stack/bigtop-manager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/hooks/InstallHookImpl.java @@ -34,7 +34,7 @@ public class InstallHookImpl implements Hook { } } - Set packages = LocalSettings.packages(); + List packages = LocalSettings.packages(); PackageUtils.install(packages); } -- Gitee From 37130d77996e7f03a1a036a06ba7cac33e4923e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E6=B2=BB=E5=9B=BD?= Date: Sun, 4 Feb 2024 17:08:38 +0800 Subject: [PATCH 4/4] 1 --- .../server/command/job/runner/AbstractJobRunner.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/AbstractJobRunner.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/AbstractJobRunner.java index 38613f5c..ff344ed3 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/AbstractJobRunner.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/runner/AbstractJobRunner.java @@ -77,6 +77,9 @@ public abstract class AbstractJobRunner implements JobRunner { @Override public void onFailure() { + job.setState(JobState.FAILED); + jobRepository.save(job); + for (Stage stage : job.getStages()) { if (stage.getState() == JobState.PENDING) { stage.setState(JobState.CANCELED); @@ -88,9 +91,6 @@ public abstract class AbstractJobRunner implements JobRunner { } } } - - job.setState(JobState.FAILED); - jobRepository.save(job); } protected CommandDTO getCommandDTO() { -- Gitee