Skip to content

Commit

Permalink
Collex to case pub sub (#253)
Browse files Browse the repository at this point in the history
* trigger GitHub actions

* Initial commit

* fix test case

* changes for security vulnerability

* auto patch increment

* Adding deployment

* updating chart version

* updating wrong name on deployment.yaml

* Adding missing google key mount

* Adding missing variables

Co-authored-by: Amit Sinha <[email protected]>
Co-authored-by: ras-rm-pr-bot <[email protected]>
  • Loading branch information
3 people authored Jun 3, 2021
1 parent 00e3c1c commit db15923
Show file tree
Hide file tree
Showing 21 changed files with 422 additions and 100 deletions.
4 changes: 2 additions & 2 deletions _infra/helm/collection-exercise/Chart.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@ type: application

# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
version: 1.3.1
version: 1.4.0

# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application.

appVersion: 11.1.2
appVersion: 11.1.3
18 changes: 16 additions & 2 deletions _infra/helm/collection-exercise/templates/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -21,16 +21,19 @@ spec:
helmVersion: {{ .Chart.Version }}
env: {{ .Values.env }}
spec:
{{- if .Values.database.sqlProxyEnabled }}
volumes:
- name: google-cloud-key
secret:
secretName: google-application-credentials
{{- if .Values.database.sqlProxyEnabled }}
- name: cloudsql-instance-credentials
secret:
secretName: cloudsql-proxy-credentials
defaultMode: 0444
items:
- key: "credentials.json"
path: "credentials.json"
{{- end }}
{{- end }}
containers:
{{- if .Values.database.sqlProxyEnabled }}
- name: cloudsql-proxy
Expand Down Expand Up @@ -67,6 +70,9 @@ spec:
image: "{{ .Values.image.devRepo }}/{{ .Chart.Name }}:{{ .Values.image.tag }}"
{{- end}}
imagePullPolicy: {{ .Values.image.pullPolicy }}
volumeMounts:
- name: google-cloud-key
mountPath: /var/secrets/google
ports:
- name: http-server
containerPort: {{ .Values.container.port }}
Expand Down Expand Up @@ -289,5 +295,13 @@ spec:
value: "{{ .Values.tomcat.maxIdle }}"
- name: SPRING_DATASOURCE_TOMCAT_MIN_IDLE
value: "{{ .Values.tomcat.minIdle }}"
- name: GOOGLE_APPLICATION_CREDENTIALS
value: /var/secrets/google/credentials.json
- name: GOOGLE_CLOUD_PROJECT
value: "{{ .Values.gcp.project }}"
- name: GCP_PROJECT
value: "{{ .Values.gcp.project }}"
- name: GCP_CASENOTIFICATIONTOPIC
value: "{{ .Values.gcp.caseNotificationTopic }}"
resources:
{{- toYaml .Values.resources.application | nindent 12 }}
6 changes: 5 additions & 1 deletion _infra/helm/collection-exercise/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -73,4 +73,8 @@ tomcat:

dns:
enabled: false
wellKnownPort: 8080
wellKnownPort: 8080

gcp:
project: ras-rm-sandbox
caseNotificationTopic: "case-notification-topic"
30 changes: 8 additions & 22 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,13 @@ services:
image: redis:3.2.9
ports:
- "17379:6379"
pubsub-emulator:
container_name: pubsub-emulator-it
image: eu.gcr.io/ons-rasrmbs-management/pubsub-emulator
ports:
- "18681:8681"
environment:
- PUBSUB_PROJECT1=test
rabbitmq:
container_name: rabbitmq-it
image: rabbitmq:3.6.10-management
Expand All @@ -30,25 +37,4 @@ services:
environment:
- DATABASE_URL=postgres://postgres:postgres@postgres-it:5432/postgres?sslmode=disable
- security_user_name=admin
- security_user_password=secret
action:
container_name: action-it
image: eu.gcr.io/ons-rasrmbs-management/action
ports:
- "38151:8151"
external_links:
- postgres-it
- redis-it
- rabbitmq-it
environment:
- spring_datasource_url=jdbc:postgresql://postgres-it:5432/postgres?sslmode=disable
- spring_datasource_username=postgres
- spring_datasource_password=postgres
- spring_liquibase_url=jdbc:postgresql://postgres-it:5432/postgres?sslmode=disable
- spring_liquibase_user=postgres
- spring_liquibase_password=postgres
- security_user_name=admin
- security_user_password=secret
- data_grid_address=redis-it:6379
- rabbitmq_host=rabbitmq-it
- rabbitmq_port=5672
- security_user_password=secret
22 changes: 20 additions & 2 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,13 @@
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>libraries-bom</artifactId>
<version>20.2.0</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>

Expand Down Expand Up @@ -214,7 +221,7 @@
<dependency>
<groupId>com.thoughtworks.xstream</groupId>
<artifactId>xstream</artifactId>
<version>1.4.15</version>
<version>1.4.17</version>
</dependency>

<dependency>
Expand Down Expand Up @@ -290,6 +297,10 @@
<artifactId>activation</artifactId>
<version>1.1.1</version>
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-pubsub</artifactId>
</dependency>
<!-- Third party end -->

<!-- Testing -->
Expand All @@ -304,7 +315,14 @@
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.6</version>
<version>2.7</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>com.github.stefanbirkner</groupId>
<artifactId>system-rules</artifactId>
<version>1.19.0</version>
<scope>test</scope>
</dependency>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,4 +26,5 @@ public class AppConfig {
private SwaggerSettings swaggerSettings;
private Rabbitmq rabbitmq;
private Logging logging;
private GCP gcp;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
package uk.gov.ons.ctp.response.collection.exercise.config;

import lombok.Data;

/** Config POJO for GCP params */
@Data
public class GCP {
String project;
String caseNotificationTopic;
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
import uk.gov.ons.ctp.response.collection.exercise.domain.CollectionExercise;
import uk.gov.ons.ctp.response.collection.exercise.domain.ExerciseSampleUnit;
import uk.gov.ons.ctp.response.collection.exercise.domain.ExerciseSampleUnitGroup;
import uk.gov.ons.ctp.response.collection.exercise.lib.casesvc.message.sampleunitnotification.SampleUnitParent;
import uk.gov.ons.ctp.response.collection.exercise.lib.common.distributed.DistributedListManager;
import uk.gov.ons.ctp.response.collection.exercise.lib.common.distributed.LockingException;
import uk.gov.ons.ctp.response.collection.exercise.lib.common.error.CTPException;
Expand All @@ -40,6 +39,7 @@
import uk.gov.ons.ctp.response.collection.exercise.representation.SampleUnitGroupDTO;
import uk.gov.ons.ctp.response.collection.exercise.representation.SampleUnitGroupDTO.SampleUnitGroupEvent;
import uk.gov.ons.ctp.response.collection.exercise.representation.SampleUnitGroupDTO.SampleUnitGroupState;
import uk.gov.ons.ctp.response.collection.exercise.representation.SampleUnitParentDTO;
import uk.gov.ons.ctp.response.collection.exercise.service.EventService;

/** Class responsible for business logic to distribute SampleUnits. */
Expand Down Expand Up @@ -197,7 +197,7 @@ private List<ExerciseSampleUnitGroup> retrieveSampleUnitGroups(CollectionExercis
private void distributeSampleUnitGroup(
CollectionExercise exercise, ExerciseSampleUnitGroup sampleUnitGroup) throws CTPException {
ExerciseSampleUnit sampleUnit = sampleUnitRepo.findBySampleUnitGroup(sampleUnitGroup).get(0);
SampleUnitParent sampleUnitParent;
SampleUnitParentDTO sampleUnitParent;

boolean activeEnrolment = hasActiveEnrolment(sampleUnit, exercise);
sampleUnitParent = sampleUnit.toSampleUnitParent(activeEnrolment, exercise.getId());
Expand Down Expand Up @@ -244,7 +244,7 @@ private boolean enrolmentIsEnabledForSurvey(final Enrolment enrolment, String su
* @param sampleUnitMessage to publish.
*/
private void publishSampleUnitToCase(
ExerciseSampleUnitGroup sampleUnitGroup, SampleUnitParent sampleUnitMessage) {
ExerciseSampleUnitGroup sampleUnitGroup, SampleUnitParentDTO sampleUnitMessage) {

transactionTemplate.execute(
new TransactionCallbackWithoutResult() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,9 @@
import net.sourceforge.cobertura.CoverageIgnore;
import org.hibernate.annotations.GenericGenerator;
import org.hibernate.annotations.Parameter;
import uk.gov.ons.ctp.response.collection.exercise.lib.casesvc.message.sampleunitnotification.SampleUnit;
import uk.gov.ons.ctp.response.collection.exercise.lib.casesvc.message.sampleunitnotification.SampleUnitParent;
import uk.gov.ons.ctp.response.collection.exercise.lib.sample.representation.SampleUnitDTO;
import uk.gov.ons.ctp.response.collection.exercise.representation.SampleUnit;
import uk.gov.ons.ctp.response.collection.exercise.representation.SampleUnitParentDTO;

/** Domain model object for sample units. */
@CoverageIgnore
Expand Down Expand Up @@ -61,9 +61,9 @@ public class ExerciseSampleUnit {
@Column(name = "sampleunittypefk")
private SampleUnitDTO.SampleUnitType sampleUnitType;

public SampleUnitParent toSampleUnitParent(
public SampleUnitParentDTO toSampleUnitParent(
final boolean activeEnrolment, final UUID collectionExerciseId) {
final SampleUnitParent parent = new SampleUnitParent();
final SampleUnitParentDTO parent = new SampleUnitParentDTO();
populateSampleUnit(activeEnrolment, parent);
parent.setCollectionExerciseId(collectionExerciseId.toString());

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@
@AllArgsConstructor
@NoArgsConstructor
public class RestUtilityConfig {
private String scheme = "http";
private String host = "localhost";
private String port = "8080";
@Builder.Default private String scheme = "http";
@Builder.Default private String host = "localhost";
@Builder.Default private String port = "8080";
private String username;
private String password;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
package uk.gov.ons.ctp.response.collection.exercise.message;

import com.godaddy.logging.Logger;
import com.godaddy.logging.LoggerFactory;
import com.google.cloud.pubsub.v1.Publisher;
import com.google.pubsub.v1.TopicName;
import java.io.IOException;
import jodd.util.StringUtil;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import uk.gov.ons.ctp.response.collection.exercise.config.AppConfig;
import uk.gov.ons.ctp.response.collection.exercise.utility.PubSubEmulator;

@Component
public class PubSub {
private static final Logger log = LoggerFactory.getLogger(PubSub.class);
@Autowired AppConfig appConfig;

private Publisher publisherSupplier(String project, String topic) throws IOException {
log.info("creating pubsub publish for topic " + topic + " in project " + project);
TopicName topicName = TopicName.of(project, topic);
if (StringUtil.isBlank(System.getenv("PUBSUB_EMULATOR_HOST"))) {
log.info("Returning actual Publisher");
return Publisher.newBuilder(topicName).build();
} else {
log.with("PubSub emulator host", System.getenv("PUBSUB_EMULATOR_HOST"))
.info("Returning emulator Publisher");
log.info("Returning emulator Publisher");
return new PubSubEmulator().getEmulatorPublisher(topicName);
}
}

public Publisher sampleUnitPublisher() throws IOException {
return publisherSupplier(
appConfig.getGcp().getProject(), appConfig.getGcp().getCaseNotificationTopic());
}

public void shutdown() {
if (StringUtil.isEmpty(System.getenv("PUBSUB_EMULATOR_HOST"))) {
PubSubEmulator.CHANNEL.shutdown();
}
}
}
Original file line number Diff line number Diff line change
@@ -1,34 +1,81 @@
package uk.gov.ons.ctp.response.collection.exercise.message;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.godaddy.logging.Logger;
import com.godaddy.logging.LoggerFactory;
import net.sourceforge.cobertura.CoverageIgnore;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.integration.annotation.MessageEndpoint;
import uk.gov.ons.ctp.response.collection.exercise.lib.casesvc.message.sampleunitnotification.SampleUnitParent;
import com.google.api.core.ApiFuture;
import com.google.api.core.ApiFutureCallback;
import com.google.api.core.ApiFutures;
import com.google.api.gax.rpc.ApiException;
import com.google.cloud.pubsub.v1.Publisher;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.protobuf.ByteString;
import com.google.pubsub.v1.PubsubMessage;
import java.io.IOException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import uk.gov.ons.ctp.response.collection.exercise.config.AppConfig;
import uk.gov.ons.ctp.response.collection.exercise.representation.SampleUnitParentDTO;

/** Service implementation responsible for publishing a sampleUnit message to the case service. */
@CoverageIgnore
@MessageEndpoint
@Service
public class SampleUnitPublisher {
private static final Logger log = LoggerFactory.getLogger(SampleUnitPublisher.class);

private RabbitTemplate rabbitTemplate;
@Autowired AppConfig appConfig;

public SampleUnitPublisher(@Qualifier("caseRabbitTemplate") RabbitTemplate rabbitTemplate) {
this.rabbitTemplate = rabbitTemplate;
}
@Autowired private PubSub pubSub;

@Autowired private ObjectMapper objectMapper;

/**
* To publish a SampleUnitGroup
*
* @param sampleUnit the SampleUnitGroup message to publish.
*/
public void sendSampleUnit(SampleUnitParent sampleUnit) {
public void sendSampleUnit(SampleUnitParentDTO sampleUnit) {
log.with("sample_unit_ref", sampleUnit.getSampleUnitRef())
.with("sample_unit_type", sampleUnit.getSampleUnitType())
.debug("Entering sendSampleUnit");
rabbitTemplate.convertAndSend(sampleUnit);
try {
String message = objectMapper.writeValueAsString(sampleUnit);
ByteString data = ByteString.copyFromUtf8(message);
PubsubMessage pubsubMessage = PubsubMessage.newBuilder().setData(data).build();
Publisher publisher = pubSub.sampleUnitPublisher();
try {
log.with("publisher", publisher).info("Publishing message to PubSub");
ApiFuture<String> messageIdFuture = publisher.publish(pubsubMessage);
ApiFutures.addCallback(
messageIdFuture,
new ApiFutureCallback<>() {
@Override
public void onFailure(Throwable throwable) {
if (throwable instanceof ApiException) {
ApiException apiException = ((ApiException) throwable);
log.with("error", apiException.getStatusCode().getCode())
.error("SampleUnit publish sent failure to PubSub.");
}
log.with("message", message).error("Error Publishing PubSub message");
}

@Override
public void onSuccess(String messageId) {
// Once published, returns server-assigned message ids (unique within the topic)
log.with("messageId", messageId).info("SampleUnit publish sent successfully");
}
},
MoreExecutors.directExecutor());
} finally {
publisher.shutdown();
pubSub.shutdown();
}
} catch (JsonProcessingException e) {
log.with("sampleUnit", sampleUnit).error("Error while sampleUnit can not be parsed.");
throw new RuntimeException(e);
} catch (IOException e) {
log.error("PubSub Error while processing sample unit distribution", e);
throw new RuntimeException(e);
}
}
}
Loading

0 comments on commit db15923

Please sign in to comment.