Merge branch '101-add-new-endpoint-to-retrieve-all-commands-from-a-training-run' into 'develop'

Resolve "Add new endpoint to retrieve all commands from a training run"

See merge request !112

src/main/java/cz/muni/ics/kypo/training/adaptive/controller/VisualizationRestController.java

@@ -2,6 +2,7 @@ package cz.muni.ics.kypo.training.adaptive.controller;
 
 
 import cz.muni.ics.kypo.training.adaptive.domain.simulator.InstanceModelUpdate;
+import cz.muni.ics.kypo.training.adaptive.dto.CommandDTO;
 import cz.muni.ics.kypo.training.adaptive.dto.trainingdefinition.TrainingDefinitionMitreTechniquesDTO;
 import cz.muni.ics.kypo.training.adaptive.dto.visualizations.sankey.SankeyDiagramDTO;
 import cz.muni.ics.kypo.training.adaptive.dto.visualizations.simulator.InstanceSimulatorDTO;
@@ -207,8 +208,8 @@ public class VisualizationRestController {
             @ApiResponse(code = 404, message = "Training run with given id not found.", response = ApiError.class),
             @ApiResponse(code = 500, message = "Unexpected condition was encountered.", response = ApiError.class)
     })
-    @GetMapping(path = "/training-runs/{runId}/commands", produces = MediaType.APPLICATION_JSON_VALUE)
-    public ResponseEntity<List<Map<String, Object>>> getAllCommandsInTrainingRun(
+    @GetMapping(path = "/commands/training-runs/{runId}/all", produces = MediaType.APPLICATION_JSON_VALUE)
+    public ResponseEntity<List<CommandDTO>> getAllCommandsInTrainingRun(
             @ApiParam(value = "Training run ID", required = true) @PathVariable("runId") Long runId) {
         return ResponseEntity.ok(visualizationFacade.getAllCommandsInTrainingRun(runId));
     }

src/main/java/cz/muni/ics/kypo/training/adaptive/converter/DurationConverter.java

+package cz.muni.ics.kypo.training.adaptive.converter;
+
+import com.fasterxml.jackson.databind.util.StdConverter;
+
+import java.time.Duration;
+
+public class DurationConverter extends StdConverter<Duration, String> {
+
+    @Override
+    public String convert(Duration duration) {
+        long seconds = duration.getSeconds();
+        long HH = seconds / 3600;
+        long MM = (seconds % 3600) / 60;
+        long SS = seconds % 60;
+        return String.format("%02d:%02d:%02d", HH, MM, SS);
+    }
+}
\ No newline at end of file

src/main/java/cz/muni/ics/kypo/training/adaptive/dto/AbstractCommandDTO.java

+package cz.muni.ics.kypo.training.adaptive.dto;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonSubTypes;
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import javax.validation.constraints.NotEmpty;
+
+@ApiModel(value = "AbstractCommandDTO", subTypes = {CommandDTO.class},
+        description = "CommandDTO")
+@JsonSubTypes({
+        @JsonSubTypes.Type(value = CommandDTO.class, name = "CommandDTO")})
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public abstract class AbstractCommandDTO {
+
+    @ApiModelProperty(value = "Distinguish tool/command line  in which command ran.", required = true, example = "BASH")
+    @NotEmpty
+    @JsonProperty("command_type")
+    public String commandType;
+
+    @ApiModelProperty(value = "Command without arguments/options.", required = true, example = "ls")
+    @NotEmpty
+    public String cmd;
+}

src/main/java/cz/muni/ics/kypo/training/adaptive/dto/CommandDTO.java

+package cz.muni.ics.kypo.training.adaptive.dto;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import cz.muni.ics.kypo.training.adaptive.converter.DurationConverter;
+import cz.muni.ics.kypo.training.adaptive.converter.LocalDateTimeUTCDeserializer;
+import cz.muni.ics.kypo.training.adaptive.converter.LocalDateTimeUTCSerializer;
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+import lombok.*;
+
+import javax.validation.constraints.NotEmpty;
+import javax.validation.constraints.NotNull;
+import javax.validation.constraints.Past;
+import java.time.Duration;
+import java.time.LocalDateTime;
+
+@Data
+@EqualsAndHashCode(callSuper = false)
+@AllArgsConstructor
+@NoArgsConstructor
+@ApiModel(value = "CommandDTO", description = "Command with valid syntax and semantic.", parent = AbstractCommandDTO.class)
+@JsonPropertyOrder({"commandType", "cmd", "timestamp", "trainingTime", "fromHostIp", "options"})
+public class CommandDTO extends AbstractCommandDTO {
+
+    @Past
+    @NotNull
+    @ApiModelProperty(value = "Time when command was recorded.", required = true, example = "2022-07-21T13:16:41.435559")
+    @JsonSerialize(using = LocalDateTimeUTCSerializer.class)
+    @JsonDeserialize(using = LocalDateTimeUTCDeserializer.class)
+    private LocalDateTime timestamp;
+    @ApiModelProperty(value = "Training time when command was recorded.", required = true, example = "07:23:43")
+    @JsonSerialize(converter = DurationConverter.class)
+    @JsonProperty("training_time")
+    private Duration trainingTime;
+    @NotEmpty
+    @ApiModelProperty(value = "Ip address where command was submitted.", required = true, example = "10.10.17.5")
+    @JsonProperty("from_host_ip")
+    private String fromHostIp;
+    @NotEmpty
+    @ApiModelProperty(value = "Valid command options", required = true, example = "-p 25 -a 20.20.15.18")
+    private String options;
+
+    @Builder
+    public CommandDTO(@NotEmpty String commandType, @NotEmpty String cmd, @Past @NotNull LocalDateTime timestamp, @NotNull Duration trainingTime, @NotEmpty String fromHostIp, @NotEmpty String options) {
+        super(commandType, cmd);
+        this.timestamp = timestamp;
+        this.trainingTime = trainingTime;
+        this.fromHostIp = fromHostIp;
+        this.options = options;
+    }
+
+}

src/main/java/cz/muni/ics/kypo/training/adaptive/facade/VisualizationFacade.java

 package cz.muni.ics.kypo.training.adaptive.facade;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import cz.muni.ics.kypo.training.adaptive.annotations.transactions.TransactionalRO;
 import cz.muni.ics.kypo.training.adaptive.domain.ParticipantTaskAssignment;
 import cz.muni.ics.kypo.training.adaptive.domain.phase.AbstractPhase;
@@ -8,7 +9,9 @@ import cz.muni.ics.kypo.training.adaptive.annotations.security.IsTrainee;
 import cz.muni.ics.kypo.training.adaptive.domain.phase.MitreTechnique;
 import cz.muni.ics.kypo.training.adaptive.domain.training.TrainingDefinition;
 import cz.muni.ics.kypo.training.adaptive.domain.training.TrainingInstance;
+import cz.muni.ics.kypo.training.adaptive.domain.training.TrainingRun;
 import cz.muni.ics.kypo.training.adaptive.dto.AbstractPhaseDTO;
+import cz.muni.ics.kypo.training.adaptive.dto.CommandDTO;
 import cz.muni.ics.kypo.training.adaptive.dto.UserRefDTO;
 import cz.muni.ics.kypo.training.adaptive.dto.questionnaire.QuestionnairePhaseDTO;
 import cz.muni.ics.kypo.training.adaptive.dto.training.TrainingPhaseDTO;
@@ -27,12 +30,17 @@ import cz.muni.ics.kypo.training.adaptive.service.phases.PhaseService;
 import cz.muni.ics.kypo.training.adaptive.service.training.TrainingDefinitionService;
 import cz.muni.ics.kypo.training.adaptive.service.training.TrainingInstanceService;
 import cz.muni.ics.kypo.training.adaptive.service.training.TrainingRunService;
+import cz.muni.ics.kypo.training.adaptive.utils.AbstractCommandPrefixes;
+import cz.muni.ics.kypo.training.adaptive.utils.ElasticSearchCommand;
 import org.springframework.data.domain.PageRequest;
 import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 
 
+import java.time.Duration;
+import java.time.LocalDateTime;
+import java.time.ZonedDateTime;
 import java.util.*;
 import java.util.function.Function;
 import java.util.function.Supplier;
@@ -49,6 +57,7 @@ public class VisualizationFacade {
     private final UserManagementServiceApi userManagementServiceApi;
     private final PhaseService phaseService;
     private final PhaseMapper phaseMapper;
+    private final ObjectMapper objectMapper;
 
     public VisualizationFacade(VisualizationService visualizationService,
                                TrainingDefinitionService trainingDefinitionService,
@@ -56,7 +65,7 @@ public class VisualizationFacade {
                                TrainingRunService trainingRunService,
                                UserManagementServiceApi userManagementServiceApi,
                                PhaseService phaseService,
-                               PhaseMapper phaseMapper) {
+                               PhaseMapper phaseMapper, ObjectMapper objectMapper) {
         this.visualizationService = visualizationService;
         this.trainingDefinitionService = trainingDefinitionService;
         this.trainingInstanceService = trainingInstanceService;
@@ -64,6 +73,7 @@ public class VisualizationFacade {
         this.userManagementServiceApi = userManagementServiceApi;
         this.phaseService = phaseService;
         this.phaseMapper = phaseMapper;
+        this.objectMapper = objectMapper;
     }
 
     @PreAuthorize("hasAuthority(T(cz.muni.ics.kypo.training.adaptive.enums.RoleTypeSecurity).ROLE_ADAPTIVE_TRAINING_ADMINISTRATOR)" +
@@ -129,10 +139,48 @@ public class VisualizationFacade {
             "or @securityService.isOrganizerOfGivenTrainingRun(#runId) " +
             "or @securityService.isTraineeOfGivenTrainingRun(#runId)")
     @TransactionalRO
-    public List<Map<String, Object>> getAllCommandsInTrainingRun(Long runId) {
-        return trainingRunService.getCommandsByTrainingRun(runId);
+    public List<CommandDTO> getAllCommandsInTrainingRun(Long runId) {
+        TrainingRun trainingRun = trainingRunService.findById(runId);
+        List<Map<String, Object>> runCommands = trainingRunService.getCommandsByTrainingRun(runId);
+        return elasticCommandsToCommandDTOlist(runCommands, trainingRun.getStartTime());
     }
 
+    private List<CommandDTO> elasticCommandsToCommandDTOlist(List<Map<String, Object>> elasticCommands, LocalDateTime runStartTime) {
+        List<CommandDTO> commandDTOS = new ArrayList<>(elasticCommands.size());
+        elasticCommands.stream()
+                .map(elasticCommand -> objectMapper.convertValue(elasticCommand, ElasticSearchCommand.class))
+                .forEach(elasticCommand -> commandDTOS.add(elasticCommandToCommandDTO(elasticCommand, runStartTime)));
+        return commandDTOS;
+    }
+
+    private CommandDTO elasticCommandToCommandDTO(ElasticSearchCommand elasticSearchCommand, LocalDateTime runStartTime) {
+        String[] commandSplit =  elasticSearchCommand.getCmd().split(" ", 2);
+        String command = commandSplit[0];
+        if (AbstractCommandPrefixes.isPrefix(command)) {
+            commandSplit = commandSplit[1].split(" ", 2);
+            command += " " + commandSplit[0];
+        }
+
+        // if there were no options, the option string should be null
+        String options = null;
+        if (commandSplit.length == 2) {
+            options = commandSplit[1];
+        }
+        String timestampString = elasticSearchCommand.getTimestampStr();
+        LocalDateTime commandTimestamp = ZonedDateTime.parse(timestampString).toLocalDateTime();
+
+        return CommandDTO.builder()
+                .commandType(elasticSearchCommand.getCmdType())
+                .cmd(command)
+                .timestamp(commandTimestamp)
+                .trainingTime(Duration.between(runStartTime, commandTimestamp))
+                .fromHostIp(elasticSearchCommand.getIp())
+                .options(options)
+                .build();
+    }
+
+
+
 
     private void removeCorrectnessFromTransitionsDataOfTrainee(TransitionsDataDTO transitionsDataDTO) {
         Map<Long, Long> visitedTasksByPhaseId = transitionsDataDTO.getTrainingRunsData().get(0).getTrainingRunPathNodes().stream()

src/main/java/cz/muni/ics/kypo/training/adaptive/utils/AbstractCommandPrefixes.java

+package cz.muni.ics.kypo.training.adaptive.utils;
+
+import java.util.Set;
+
+/**
+ * Prefixes which should be included with the command. Example: sudo ls
+ */
+public abstract class AbstractCommandPrefixes {
+    private static final Set<String> PREFIXES = Set.of("sudo");
+
+    public static boolean isPrefix(String prefix) {
+        return PREFIXES.contains(prefix);
+    }
+}

src/main/java/cz/muni/ics/kypo/training/adaptive/utils/ElasticSearchCommand.java

+package cz.muni.ics.kypo.training.adaptive.utils;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.Objects;
+
+/**
+ * This class is a representation of a logged command retrieved by the elastic search api.
+ */
+public class ElasticSearchCommand {
+    private String hostname;
+    private String ip;
+    @JsonProperty("timestamp_str")
+    private String timestampStr;
+    @JsonProperty("sandbox_id")
+    private Long sandboxId;
+    private String cmd;
+    @JsonProperty("pool_id")
+    private Long poolId;
+    private String wd;
+    @JsonProperty("cmd_type")
+    private String cmdType;
+    private String username;
+
+    public String getHostname() {
+        return hostname;
+    }
+
+    public void setHostname(String hostname) {
+        this.hostname = hostname;
+    }
+
+    public String getIp() {
+        return ip;
+    }
+
+    public void setIp(String ip) {
+        this.ip = ip;
+    }
+
+    public String getTimestampStr() {
+        return timestampStr;
+    }
+
+    public void setTimestampStr(String timestampStr) {
+        this.timestampStr = timestampStr;
+    }
+
+    public Long getSandboxId() {
+        return sandboxId;
+    }
+
+    public void setSandboxId(Long sandboxId) {
+        this.sandboxId = sandboxId;
+    }
+
+    public String getCmd() {
+        return cmd;
+    }
+
+    public void setCmd(String cmd) {
+        this.cmd = cmd;
+    }
+
+    public Long getPoolId() {
+        return poolId;
+    }
+
+    public void setPoolId(Long poolId) {
+        this.poolId = poolId;
+    }
+
+    public String getWd() {
+        return wd;
+    }
+
+    public void setWd(String wd) {
+        this.wd = wd;
+    }
+
+    public String getCmdType() {
+        return cmdType;
+    }
+
+    public void setCmdType(String cmdType) {
+        this.cmdType = cmdType;
+    }
+
+    public String getUsername() {
+        return username;
+    }
+
+    public void setUsername(String username) {
+        this.username = username;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        ElasticSearchCommand that = (ElasticSearchCommand) o;
+        return hostname.equals(that.hostname) && ip.equals(that.ip) &&
+                timestampStr.equals(that.timestampStr) &&
+                sandboxId.equals(that.sandboxId) && cmd.equals(that.cmd) &&
+                poolId.equals(that.poolId) && wd.equals(that.wd) &&
+                cmdType.equals(that.cmdType) &&
+                username.equals(that.username);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(hostname, ip, timestampStr, sandboxId, cmd, poolId, wd, cmdType, username);
+    }
+
+    @Override
+    public String toString() {
+        return "ElasticSearchCommand{" +
+                "hostname='" + hostname + '\'' +
+                ", ip='" + ip + '\'' +
+                ", timestampStr='" + timestampStr + '\'' +
+                ", sandboxId=" + sandboxId +
+                ", cmd='" + cmd + '\'' +
+                ", poolId=" + poolId +
+                ", wd='" + wd + '\'' +
+                ", cmdType='" + cmdType + '\'' +
+                ", username='" + username + '\'' +
+                '}';
+    }
+}
+

src/test/java/cz/muni/ics/kypo/training/adaptive/facade/VisualizationFacadeTest.java

+package cz.muni.ics.kypo.training.adaptive.facade;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import cz.muni.ics.kypo.training.adaptive.domain.training.TrainingInstance;
+import cz.muni.ics.kypo.training.adaptive.domain.training.TrainingRun;
+import cz.muni.ics.kypo.training.adaptive.dto.CommandDTO;
+import cz.muni.ics.kypo.training.adaptive.exceptions.EntityNotFoundException;
+import cz.muni.ics.kypo.training.adaptive.mapping.PhaseMapper;
+import cz.muni.ics.kypo.training.adaptive.service.VisualizationService;
+import cz.muni.ics.kypo.training.adaptive.service.api.UserManagementServiceApi;
+import cz.muni.ics.kypo.training.adaptive.service.phases.PhaseService;
+import cz.muni.ics.kypo.training.adaptive.service.training.TrainingDefinitionService;
+import cz.muni.ics.kypo.training.adaptive.service.training.TrainingInstanceService;
+import cz.muni.ics.kypo.training.adaptive.service.training.TrainingRunService;
+import cz.muni.ics.kypo.training.adaptive.util.TestDataFactory;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.MockitoAnnotations;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.mock.mockito.MockBean;
+
+import java.time.Duration;
+import java.time.LocalDateTime;
+import java.util.*;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.BDDMockito.given;
+
+@SpringBootTest(classes = {
+        TestDataFactory.class,
+        ObjectMapper.class
+})
+class VisualizationFacadeTest {
+
+    private VisualizationFacade visualizationFacade;
+
+    @Autowired
+    private TestDataFactory testDataFactory;
+    @Autowired
+    private ObjectMapper objectMapper;
+
+    @MockBean
+    private VisualizationService visualizationService;
+    @MockBean
+    private TrainingDefinitionService trainingDefinitionService;
+    @MockBean
+    private TrainingInstanceService trainingInstanceService;
+    @MockBean
+    private TrainingRunService trainingRunService;
+    @MockBean
+    private UserManagementServiceApi userManagementServiceApi;
+    @MockBean
+    private PhaseService phaseService;
+    @MockBean
+    private PhaseMapper phaseMapper;
+
+    private List<Map<String, Object>> elasticCommands;
+    private TrainingRun trainingRun;
+    private TrainingInstance trainingInstance;
+    private List<CommandDTO> expected;
+
+    @BeforeEach
+    public void init() {
+        MockitoAnnotations.openMocks(this);
+        visualizationFacade = new VisualizationFacade(visualizationService, trainingDefinitionService,
+                trainingInstanceService, trainingRunService, userManagementServiceApi, phaseService, phaseMapper, objectMapper);
+
+        elasticCommands = List.of(
+                Map.of("hostname","attacker","ip","10.1.26.23","timestamp_str","2022-07-21T13:16:41.435559Z","sandbox_id","1",
+                        "cmd","sudo ls","pool_id","1","wd","/home/user","cmd_type","bash-command","username","user"),
+                Map.of("hostname","attacker","ip","10.1.26.23","timestamp_str","2022-07-21T13:16:42.276428Z","sandbox_id","1",
+                        "cmd","cat","pool_id","1","wd","/home/user","cmd_type","bash-command","username","user"),
+                Map.of("hostname","attacker","ip","10.1.26.23","timestamp_str","2022-07-21T13:16:52.658178Z","sandbox_id","1",
+                        "cmd","echo f > a","pool_id","1","wd","/home/user","cmd_type","bash-command","username","user"),
+                Map.of("hostname","attacker","ip","10.1.26.23","timestamp_str","2022-07-21T13:17:09.732708Z","sandbox_id","1","cmd",
+                        "sudo nmap -v","pool_id","1","wd","/home/user","cmd_type","bash-command","username","user")
+        );
+
+        expected = List.of(
+                new CommandDTO("bash-command", "sudo ls", LocalDateTime.parse("2022-07-21T13:16:41.435559"),
+                        Duration.parse("PT1.435559S"), "10.1.26.23", null),
+                new CommandDTO("bash-command", "cat", LocalDateTime.parse("2022-07-21T13:16:42.276428"),
+                        Duration.parse("PT2.276428S"), "10.1.26.23", null),
+                new CommandDTO("bash-command", "echo", LocalDateTime.parse("2022-07-21T13:16:52.658178"),
+                        Duration.parse("PT12.658178S"), "10.1.26.23", "f > a"),
+                new CommandDTO("bash-command", "sudo nmap", LocalDateTime.parse("2022-07-21T13:17:09.732708"),
+                        Duration.parse("PT29.732708S"), "10.1.26.23", "-v")
+        );
+
+        LocalDateTime startTime = LocalDateTime.parse("2022-07-21T13:16:40");
+
+        trainingRun = testDataFactory.getFinishedRun();
+        trainingInstance = testDataFactory.getConcludedInstance();
+        trainingRun.setStartTime(startTime);
+        trainingRun.setTrainingInstance(trainingInstance);
+        trainingInstance.setLocalEnvironment(false);
+    }
+
+    @Test
+    void getAllCommandsByTrainingRun() {
+        given(trainingRunService.findById(trainingRun.getId())).willReturn(trainingRun);
+        given(trainingRunService.getCommandsByTrainingRun(trainingRun.getId())).willReturn(elasticCommands);
+        List<CommandDTO> received = visualizationFacade.getAllCommandsInTrainingRun(trainingRun.getId());
+        assertEquals(expected.size(), received.size());
+        compareCommandDTOLists(expected, received);
+    }
+
+    @Test
+    void trainingRunDoesNotExist() {
+        given(trainingRunService.findById(anyLong())).willThrow(EntityNotFoundException.class);
+        assertThrows(EntityNotFoundException.class, () -> visualizationFacade.getAllCommandsInTrainingRun(anyLong()));
+    }
+
+    @Test
+    void noCommandsFound() {
+        given(trainingRunService.findById(anyLong())).willReturn(trainingRun);
+        trainingInstance.setLocalEnvironment(false);
+        given(trainingRunService.getCommandsByTrainingRun(trainingRun.getId())).willReturn(Collections.emptyList());
+        List<CommandDTO> received = visualizationFacade.getAllCommandsInTrainingRun(anyLong());
+        assertEquals(0, received.size());
+    }
+
+    private void compareCommandDTOLists(List<CommandDTO> list1, List<CommandDTO> list2) {
+        if (list1.size() != list2.size()) {
+            fail();
+        }
+
+        for (int i = 0; i < list1.size(); i++) {
+            compareCommands(list1.get(i), list2.get(i));
+        }
+    }
+
+    private void compareCommands(CommandDTO cmd1, CommandDTO cmd2) {
+        assertEquals(cmd1.getCmd(), cmd2.getCmd());
+        assertEquals(cmd1.getOptions(), cmd2.getOptions());
+        assertEquals(cmd1.getTimestamp(), cmd2.getTimestamp());
+        assertEquals(cmd1.getTrainingTime(), cmd2.getTrainingTime());
+        assertEquals(cmd1.getCommandType(), cmd2.getCommandType());
+        assertEquals(cmd1.getFromHostIp(), cmd2.getFromHostIp());
+    }
+}
\ No newline at end of file