Development: Switch postgres e2e setup to Jenkins/Gitlab

docker/artemis-migration-check-postgres.yml

@@ -8,7 +8,7 @@ services:
             file: ./artemis.yml
             service: artemis-app
         env_file:
-            - ./artemis/config/postgres.env
+            - ./artemis/config/cypress-postgres.env
             - ./artemis/config/migration-check.env
         depends_on:
             postgresql:

docker/artemis/config/cypress.env → docker/artemis/config/cypress-mysql.env

@@ -1,11 +1,16 @@
 SPRING_PROFILES_ACTIVE="artemis,scheduling,bamboo,bitbucket,jira,prod,docker"
 
+SPRING_DATASOURCE_URL="jdbc:mysql://artemis-mysql:3306/Artemis?createDatabaseIfNotExist=true&allowPublicKeyRetrieval=true&useUnicode=true&characterEncoding=utf8&useSSL=false&useLegacyDatetimeCode=false&serverTimezone=UTC"
 SPRING_DATASOURCE_USERNAME="root"
 SPRING_DATASOURCE_PASSWORD=""
 SPRING_DATASOURCE_HIKARI_MAXIMUMPOOLSIZE="100"
+
+SPRING_JPA_DATABASE_PLATFORM="org.hibernate.dialect.MySQL8Dialect"
+SPRING_JPA_DATABASE="MYSQL"
 SPRING_JPA_HIBERNATE_CONNECTION_CHARSET="utf8mb4"
 SPRING_JPA_HIBERNATE_CONNECTION_CHARACTERENCODING="utf8mb4"
 SPRING_JPA_HIBERNATE_CONNECTION_USEUNICODE="true"
+
 SPRING_PROMETHEUS_MONITORINGIP="131.159.89.160"
 
 # for bamboo and bitbucket notifications on /api/programming-exercises/new-result it seems like port

docker/artemis/config/cypress-postgres.env

@@ -0,0 +1,69 @@
+# ----------------------------------------------------------------------------------------------------------------------
+# Artemis configurations for Postgres setups
+# ----------------------------------------------------------------------------------------------------------------------
+
+SPRING_PROFILES_ACTIVE="artemis,scheduling,jenkins,gitlab,prod,docker"
+
+SPRING_DATASOURCE_URL="jdbc:postgresql://artemis-postgresql:5432/Artemis?sslmode=disable"
+SPRING_DATASOURCE_USERNAME="postgres"
+SPRING_DATASOURCE_PASSWORD=""
+SPRING_DATASOURCE_HIKARI_MAXIMUMPOOLSIZE="100"
+
+SPRING_JPA_DATABASE_PLATFORM="org.hibernate.dialect.PostgreSQL10Dialect"
+SPRING_JPA_DATABASE="POSTGRESQL"
+SPRING_JPA_HIBERNATE_CONNECTION_CHARSET="utf8mb4"
+SPRING_JPA_HIBERNATE_CONNECTION_CHARACTERENCODING="utf8mb4"
+SPRING_JPA_HIBERNATE_CONNECTION_USEUNICODE="true"
+
+SPRING_PROMETHEUS_MONITORINGIP="131.159.89.160"
+
+# for bamboo and bitbucket notifications on /api/programming-exercises/new-result it seems like port
+# 54321 is internally open for the bamboo agents
+# also can't use SSL for this as the hostnames are not integrated in the self-signed certificate
+SERVER_URL="http://${HOST_HOSTNAME}:54321"
+
+# We don't need secure passwords for testing. Lower rounds will speed up tests. 4 is the lowest
+ARTEMIS_BCRYPTSALTROUNDS="4"
+
+ARTEMIS_USERMANAGEMENT_USEEXTERNAL="false"
+ARTEMIS_USERMANAGEMENT_INTERNALADMIN_USERNAME="${bamboo_artemis_admin_username}"
+ARTEMIS_USERMANAGEMENT_INTERNALADMIN_PASSWORD="${bamboo_artemis_admin_password}"
+ARTEMIS_USERMANAGEMENT_LOGIN_ACCOUNTNAME="TUM"
+
+ARTEMIS_VERSIONCONTROL_URL="https://gitlab-test.artemis.in.tum.de"
+ARTEMIS_VERSIONCONTROL_USER="${bamboo_gitlab_admin_user}"
+ARTEMIS_VERSIONCONTROL_PASSWORD="${bamboo_gitlab_admin_password}"
+ARTEMIS_VERSIONCONTROL_TOKEN="${bamboo_gitlab_token_secret}"
+ARTEMIS_VERSIONCONTROL_CITOKEN="${bamboo_jenkins_token_secret}"
+ARTEMIS_VERSIONCONTROL_HEALTHAPITOKEN="${bamboo_gitlab_health_token_secret}"
+
+ARTEMIS_CONTINUOUSINTEGRATION_URL="https://jenkins-test.artemis.in.tum.de"
+ARTEMIS_CONTINUOUSINTEGRATION_USER="${bamboo_jenkins_admin_user}"
+ARTEMIS_CONTINUOUSINTEGRATION_PASSWORD="${bamboo_jenkins_admin_password}"
+ARTEMIS_CONTINUOUSINTEGRATION_SECRETPUSHTOKEN="${bamboo_jenkins_secret_push_token}"
+ARTEMIS_CONTINUOUSINTEGRATION_VCSCREDENTIALS="${bamboo_jenkins_vcs_credentials_secret}"
+ARTEMIS_CONTINUOUSINTEGRATION_ARTEMISAUTHENTICATIONTOKENKEY="${bamboo_jenkins_artemis_ci_authentication_token_key_secret}"
+ARTEMIS_CONTINUOUSINTEGRATION_ARTEMISAUTHENTICATIONTOKENVALUE="${bamboo_jenkins_artemis_ci_authentication_token_value_secret}"
+ARTEMIS_CONTINUOUSINTEGRATION_EMPTYCOMMITNECESSARY="true"
+ARTEMIS_CONTINUOUSINTEGRATION_BUILDTIMEOUT="30"
+
+ARTEMIS_APOLLON_CONVERSIONSERVICEURL="https://apollon.ase.in.tum.de/api/converter"
+
+# Token is valid 3 days
+JHIPSTER_SECURITY_AUTHENTICATION_JWT_TOKENVALIDITYINSECONDS="259200"
+# Token is valid 30 days
+JHIPSTER_SECURITY_AUTHENTICATION_JWT_TOKENVALIDITYINSECONDSFORREMEMBERME="2592000"
+
+# Properties to be exposed on the /info management endpoint
+
+INFO_IMPRINT="https://ase.in.tum.de/lehrstuhl_1/component/content/article/179-imprint"
+INFO_TESTSERVER="true"
+INFO_TEXTASSESSMENTANALYTICSENABLED="true"
+INFO_STUDENTEXAMSTORESESSIONDATA="true"
+
+LOGGING_FILE_NAME="/opt/artemis/data/artemis.log"
+
+MANAGEMENT_METRICS_EXPORT_PROMETHEUS_ENABLED="true"
+
+JENKINS_INTERNALURLS_CIURL="https://jenkins-test.artemis.in.tum.de"
+JENKINS_INTERNALURLS_VCNURL="https://gitlab-test.artemis.in.tum.de"

docker/cypress-E2E-tests-mysql.yml

@@ -16,7 +16,7 @@ services:
             mysql:
                 condition: service_healthy
         env_file:
-            - ./artemis/config/cypress.env
+            - ./artemis/config/cypress-mysql.env
 
     nginx:
         extends:

docker/cypress-E2E-tests-postgres.yml

@@ -16,8 +16,7 @@ services:
             postgresql:
                 condition: service_healthy
         env_file:
-            - ./artemis/config/cypress.env
-            - ./artemis/config/postgres.env
+            - ./artemis/config/cypress-postgres.env
 
     nginx:
         extends:

docker/cypress.yml

@@ -19,6 +19,7 @@ services:
             CYPRESS_tutorGroupName: "artemis-e2etest-tutors"
             CYPRESS_editorGroupName: "artemis-e2etest-editors"
             CYPRESS_instructorGroupName: "artemis-e2etest-instructors"
+            CYPRESS_createUsers: "${bamboo_cypress_create_users}"
             SORRY_CYPRESS_KEY: "${bamboo_sorry_cypress_record_secret}"
             SORRY_CYPRESS_URL: "${bamboo_sorry_cypress_url}"
             SORRY_CYPRESS_BUILD_ID: "${bamboo_buildNumber}"

src/main/java/de/tum/in/www1/artemis/service/connectors/gitlab/GitLabService.java

@@ -8,6 +8,7 @@
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
 import java.util.*;
+import java.util.Optional;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
@@ -59,6 +60,9 @@ public class GitLabService extends AbstractVersionControlService {
     @Value("${artemis.version-control.ci-token}")
     private String ciToken;
 
+    @Value("${artemis.version-control.health-api-token}")
+    private Optional<String> ciHealthToken;
+
     private final UserRepository userRepository;
 
     private final RestTemplate shortTimeoutRestTemplate;
@@ -504,7 +508,10 @@ private void updateMemberPermissionInRepository(VcsRepositoryUrl repositoryUrl,
     @Override
     public ConnectorHealth health() {
         try {
-            final var uri = Endpoints.HEALTH.buildEndpoint(gitlabServerUrl.toString()).build().toUri();
+            UriComponentsBuilder builder = Endpoints.HEALTH.buildEndpoint(gitlabServerUrl.toString());
+            ciHealthToken.ifPresent(token -> builder.queryParam("token", token));
+            URI uri = builder.build().toUri();
+
             final var healthResponse = shortTimeoutRestTemplate.getForObject(uri, JsonNode.class);
             final var status = healthResponse.get("status").asText();
             if (!status.equals("ok")) {

src/main/java/de/tum/in/www1/artemis/service/connectors/jenkins/JenkinsUserManagementService.java

@@ -85,7 +85,8 @@ public void createUser(User user, String password) throws ContinuousIntegrationE
         }
         // Only create a user if it doesn't already exist.
         if (getUser(user.getLogin()) != null) {
-            throw new JenkinsException("Cannot create user: " + user.getLogin() + " because the login already exists");
+            log.debug("Cannot create user: {} because the login already exists", user.getLogin());
+            return;
         }
 
         // Make sure the user login contains legal characters.

src/test/cypress/cypress.env.json

@@ -7,5 +7,6 @@
     "studentGroupName": null,
     "tutorGroupName": null,
     "editorGroupName": null,
-    "instructorGroupName": null
+    "instructorGroupName": null,
+    "createUsers": false
 }

src/test/cypress/e2e/Logout.cy.ts

@@ -13,6 +13,8 @@ describe('Logout tests', () => {
 
         courseManagementRequest.createCourse(true).then((response) => {
             course = convertModelAfterMultiPart(response);
+            courseManagementRequest.addStudentToCourse(course, studentOne);
+            courseManagementRequest.addStudentToCourse(course, studentTwo);
             courseManagementRequest.createModelingExercise({ course }).then((resp: Cypress.Response<ModelingExercise>) => {
                 modelingExercise = resp.body;
             });

src/test/cypress/e2e/exam/ExamAssessment.cy.ts

@@ -52,7 +52,7 @@ describe('Exam assessment', () => {
     // For some reason the typing of cypress gets slower the longer the test runs, so we test the programming exercise first
     describe('Programming exercise assessment', () => {
         before('Prepare exam', () => {
-            examEnd = dayjs().add(1, 'minutes');
+            examEnd = dayjs().add(2, 'minutes');
             prepareExam(course, examEnd, EXERCISE_TYPE.Programming);
         });
 

src/test/cypress/init/ImportUsers.cy.ts

@@ -1,11 +1,24 @@
-import { admin, instructor, studentOne, studentThree, studentTwo, tutor } from '../support/users';
+import { userManagementRequest } from '../support/requests/ArtemisRequests';
+import { USER_ID, USER_ROLE, admin, instructor, studentOne, studentThree, studentTwo, tutor, users } from '../support/users';
 
-it('Logs in once with all required users', () => {
-    // If Artemis hasn't imported the required users from Jira we have to force this by logging in with these users once
-    cy.login(admin);
-    cy.login(instructor);
-    cy.login(tutor);
-    cy.login(studentOne);
-    cy.login(studentTwo);
-    cy.login(studentThree);
+describe('Setup users', () => {
+    if (Cypress.env('createUsers')) {
+        before('Creates all required users', () => {
+            cy.login(admin);
+            for (const userKey in USER_ID) {
+                const user = users.getUserWithId(USER_ID[userKey]);
+                userManagementRequest.createUser(user.username, user.password, USER_ROLE[userKey]);
+            }
+        });
+    }
+
+    it('Logs in once with all required users', () => {
+        // If Artemis hasn't imported the required users from Jira we have to force this by logging in with these users once
+        cy.login(admin);
+        cy.login(instructor);
+        cy.login(tutor);
+        cy.login(studentOne);
+        cy.login(studentTwo);
+        cy.login(studentThree);
+    });
 });

src/test/cypress/support/requests/ArtemisRequests.ts

@@ -1,10 +1,13 @@
 import { CourseManagementRequests } from './CourseManagementRequests';
+import { UserManagementRequests } from './UserManagementRequest';
 
 /**
  * A class which encapsulates all cypress requests, which can be sent to Artemis.
  */
 export class ArtemisRequests {
     courseManagement = new CourseManagementRequests();
+    userManagement = new UserManagementRequests();
 }
 
 export const courseManagementRequest = new CourseManagementRequests();
+export const userManagementRequest = new UserManagementRequests();

src/test/cypress/support/requests/UserManagementRequest.ts

@@ -0,0 +1,29 @@
+import { BASE_API, POST } from '../constants';
+import { UserRole } from '../users';
+
+/**
+ * A class which encapsulates all cypress requests related to course management.
+ */
+export class UserManagementRequests {
+    /**
+     * Creates a new user
+     * @param username the username of the new user
+     * @param password the password of the new user
+     * @param role the role of the new user
+     */
+    createUser(username: string, password: string, role: UserRole) {
+        const user = {
+            login: username,
+            password,
+            firstName: username,
+            lastName: username,
+            email: username + '@example.com',
+            authorities: [role],
+        };
+        return cy.request({
+            url: BASE_API + 'admin/users',
+            method: POST,
+            body: user,
+        });
+    }
+}

src/test/cypress/support/users.ts

@@ -1,5 +1,27 @@
 import { BASE_API, GET, USER_ID_SELECTOR } from './constants';
 
+export enum UserRole {
+    Instructor = 'ROLE_INSTRUCTOR',
+    Tutor = 'ROLE_TA',
+    Student = 'ROLE_USER',
+}
+
+export const USER_ID = {
+    studentOne: 100,
+    studentTwo: 102,
+    studentThree: 104,
+    instructor: 103,
+    tutor: 101,
+};
+
+export const USER_ROLE = {
+    studentOne: UserRole.Student,
+    studentTwo: UserRole.Student,
+    studentThree: UserRole.Student,
+    instructor: UserRole.Instructor,
+    tutor: UserRole.Tutor,
+};
+
 /**
  * Class to encompass user management logic for cypress tests.
  */
@@ -17,40 +39,40 @@ export class CypressUserManagement {
      * @returns the first testing account with student rights.
      */
     public getStudentOne(): CypressCredentials {
-        return this.getUserWithId('100');
+        return this.getUserWithId(USER_ID.studentOne);
     }
 
     /**
      * @returns the second testing account with student rights.
      */
     public getStudentTwo(): CypressCredentials {
-        return this.getUserWithId('102');
+        return this.getUserWithId(USER_ID.studentTwo);
     }
 
     /**
      * @returns the third testing account with student rights.
      */
     public getStudentThree(): CypressCredentials {
-        return this.getUserWithId('104');
+        return this.getUserWithId(USER_ID.studentThree);
     }
 
     /**
      * @returns an instructor account.
      */
     public getInstructor(): CypressCredentials {
-        return this.getUserWithId('103');
+        return this.getUserWithId(USER_ID.instructor);
     }
 
     /**
      * @returns a tutor account.
      */
     public getTutor(): CypressCredentials {
-        return this.getUserWithId('101');
+        return this.getUserWithId(USER_ID.tutor);
     }
 
-    private getUserWithId(userId: string): CypressCredentials {
-        const username = this.getUsernameTemplate().replace(USER_ID_SELECTOR, userId);
-        const password = this.getPasswordTemplate().replace(USER_ID_SELECTOR, userId);
+    public getUserWithId(userId: number): CypressCredentials {
+        const username = this.getUsernameTemplate().replace(USER_ID_SELECTOR, userId.toString());
+        const password = this.getPasswordTemplate().replace(USER_ID_SELECTOR, userId.toString());
         return { username, password };
     }
 

src/test/java/de/tum/in/www1/artemis/authentication/UserJenkinsGitlabIntegrationTest.java

@@ -197,8 +197,8 @@ void createUser_asAdmin_existingEmail() throws Exception {
 
     @Test
     @WithMockUser(username = "admin", roles = "ADMIN")
-    void createUser_asAdmin_existsInCi_internalError() throws Exception {
-        userTestService.createUser_asAdmin_existsInCi_internalError();
+    void createUserAsAdminExistsInCi() throws Exception {
+        userTestService.createUserAsAdminExistsInCi();
     }
 
     @Test

src/test/java/de/tum/in/www1/artemis/connector/GitlabRequestMockProvider.java

@@ -59,6 +59,9 @@ public class GitlabRequestMockProvider {
     @Value("${artemis.version-control.url}")
     private URL gitlabServerUrl;
 
+    @Value("${artemis.version-control.health-api-token}")
+    private String ciHealthToken;
+
     private final RestTemplate restTemplate;
 
     private MockRestServiceServer mockServer;
@@ -339,7 +342,7 @@ public void mockFailToCheckIfProjectExists(String projectKey) throws GitLabApiEx
     }
 
     public void mockHealth(String healthStatus, HttpStatus httpStatus) throws URISyntaxException, JsonProcessingException {
-        final var uri = UriComponentsBuilder.fromUri(gitlabServerUrl.toURI()).path("/-/liveness").build().toUri();
+        final var uri = UriComponentsBuilder.fromUri(gitlabServerUrl.toURI()).path("/-/liveness").queryParam("token", ciHealthToken).build().toUri();
         final var response = new ObjectMapper().writeValueAsString(Map.of("status", healthStatus));
         mockServerShortTimeout.expect(requestTo(uri)).andExpect(method(HttpMethod.GET)).andRespond(withStatus(httpStatus).contentType(MediaType.APPLICATION_JSON).body(response));
     }