W-17464266: Add more unit tests

carbonj.service/build.gradle

@@ -233,7 +233,7 @@ dependencies {
 
 test {
     useJUnitPlatform()
-    maxHeapSize = "2g"
+    maxHeapSize = "3g"
     jvmArgs = [
             '--add-opens', 'java.base/java.util=ALL-UNNAMED',
     ]

carbonj.service/src/main/java/com/demandware/carbonj/service/db/model/MsgPackMetric.java

@@ -8,8 +8,11 @@
 
 public class MsgPackMetric
 {
-    final public String path;
-    final public boolean isLeaf;
+    public String path;
+    public boolean isLeaf;
+
+    public MsgPackMetric() {
+    }
 
     public MsgPackMetric( Metric metric )
     {

carbonj.service/src/main/java/com/demandware/carbonj/service/engine/DestinationGroup.java

@@ -142,6 +142,10 @@ String getDest()
         return dest;
     }
 
+    LineProtocolDestination[] getDestinations() {
+        return destinations;
+    }
+
     void close()
     {
         close( Arrays.asList(destinations) );

carbonj.service/src/test/java/com/demandware/carbonj/service/engine/TestCarbonjAdmin.java

@@ -0,0 +1,151 @@
+/**
+ * Copyright (c) 2018, salesforce.com, inc.
+ * All rights reserved.
+ * SPDX-License-Identifier: BSD-3-Clause
+ * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+ */
+package com.demandware.carbonj.service.engine;
+
+import com.demandware.carbonj.service.db.model.RetentionPolicy;
+import org.joda.time.DateTime;
+import org.junit.jupiter.api.Test;
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpMethod;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.client.HttpServerErrorException;
+import org.springframework.web.client.RestTemplate;
+import org.springframework.web.util.UriComponentsBuilder;
+
+import java.net.URI;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+
+public class TestCarbonjAdmin extends CarbonJSvcLongIdTest {
+    @Test
+    public void testCarbonjAdmin() {
+        cjClient.send( "a.b.c", 1.0f, new DateTime() );
+        cjClient.send( "a.b.d", 1.0f, new DateTime() );
+        drain();
+
+        RestTemplate restTemplate = new RestTemplate();
+        Map<String, String> queryParams = new HashMap<>();
+        queryParams.put("srcIp", "127.0.0.1");
+        queryParams.put("srcPort", "2001");
+        URI uri = UriComponentsBuilder.fromHttpUrl("http://127.0.0.1:2001/_dw/rest/carbonj/loadseriesfrom/60s24h")
+                .queryParam("srcIp", "{srcIp}")
+                .queryParam("srcPort", "{srcPort}")
+                .build(queryParams);
+        HttpEntity<String> httpEntity = new HttpEntity<>(new HttpHeaders());
+        ResponseEntity<String> response = restTemplate.exchange(uri, HttpMethod.POST, httpEntity, String.class);
+        assertEquals(HttpStatus.OK.value(), response.getStatusCode().value());
+        assertEquals("a.b.c:2 1 totalPoints:1 cursor:0 totalMetrics:1\n" +
+                "a.b.d:3 1 totalPoints:2 cursor:0 totalMetrics:2\n", response.getBody());
+
+        uri = UriComponentsBuilder.fromHttpUrl("http://127.0.0.1:2001/_dw/rest/carbonj/abortload").build().toUri();
+        response = restTemplate.exchange(uri, HttpMethod.POST, httpEntity, String.class);
+        assertEquals(HttpStatus.OK.value(), response.getStatusCode().value());
+        assertEquals("load is not running", response.getBody());
+
+        uri = UriComponentsBuilder.fromHttpUrl("http://127.0.0.1:2001/_dw/rest/carbonj/loadseries/60s24h").build().toUri();
+        response = restTemplate.exchange(uri, HttpMethod.POST, httpEntity, String.class);
+        assertEquals(HttpStatus.OK.value(), response.getStatusCode().value());
+        assertNull(response.getBody());
+
+        uri = UriComponentsBuilder.fromHttpUrl("http://127.0.0.1:2001/_dw/rest/carbonj/metrics/a.b.c").build().toUri();
+        response = restTemplate.exchange(uri, HttpMethod.GET, httpEntity, String.class);
+        assertEquals(HttpStatus.OK.value(), response.getStatusCode().value());
+        assertTrue(response.getBody().startsWith("{\"id\":2,\"name\":\"a.b.c\""));
+
+        uri = UriComponentsBuilder.fromHttpUrl("http://127.0.0.1:2001/_dw/rest/carbonj/metrics/foo.bar").build().toUri();
+        try {
+            restTemplate.exchange(uri, HttpMethod.GET, httpEntity, String.class);
+            fail("Should have thrown exception");
+        } catch (HttpServerErrorException e) {
+            assertEquals("500 Server Error: \"{\"status\":\"ERROR\",\"message\":\"Metric [foo.bar] not found.\"}\"", e.getMessage());
+        }
+
+        uri = UriComponentsBuilder.fromHttpUrl("http://127.0.0.1:2001/_dw/rest/carbonj/metricsearch").build().toUri();
+        try {
+            restTemplate.exchange(uri, HttpMethod.GET, httpEntity, String.class);
+            fail("Should have thrown exception");
+        } catch (HttpServerErrorException e) {
+            assertEquals("500 Server Error: \"{\"status\":\"ERROR\",\"message\":\"request parameter 'metricId' should be a positive int, Actual value [0]\"}\"", e.getMessage());
+        }
+
+        uri = UriComponentsBuilder.fromHttpUrl("http://127.0.0.1:2001/_dw/rest/carbonj/metricsearch?metricId=1234").build().toUri();
+        try {
+            restTemplate.exchange(uri, HttpMethod.GET, httpEntity, String.class);
+            fail("Should have thrown exception");
+        } catch (HttpServerErrorException e) {
+            assertEquals("500 Server Error: \"{\"status\":\"ERROR\",\"message\":\"Metric with id [1234] not found.\"}\"", e.getMessage());
+        }
+
+        uri = UriComponentsBuilder.fromHttpUrl("http://127.0.0.1:2001/_dw/rest/carbonj/metricsearch?metricId=2").build().toUri();
+        response = restTemplate.exchange(uri, HttpMethod.GET, httpEntity, String.class);
+        assertEquals(HttpStatus.OK.value(), response.getStatusCode().value());
+        assertTrue(response.getBody().startsWith("[{\"id\":2,\"name\":\"a.b.c\""));
+
+        uri = UriComponentsBuilder.fromHttpUrl("http://127.0.0.1:2001/_dw/rest/carbonj/getIdStoreName/a.b.c").build().toUri();
+        response = restTemplate.exchange(uri, HttpMethod.GET, httpEntity, String.class);
+        assertEquals(HttpStatus.OK.value(), response.getStatusCode().value());
+        assertEquals("a.b.c", response.getBody());
+
+        uri = UriComponentsBuilder.fromHttpUrl("http://127.0.0.1:2001/_dw/rest/carbonj/getIdStoreName/foo.bar").build().toUri();
+        try {
+            restTemplate.exchange(uri, HttpMethod.GET, httpEntity, String.class);
+            fail("Should have thrown exception");
+        } catch (HttpServerErrorException e) {
+            assertEquals("500 Server Error: \"{\"status\":\"ERROR\",\"message\":\"Metric [foo.bar] not found.\"}\"", e.getMessage());
+        }
+
+        int current = (int) (System.currentTimeMillis() / 1000);
+        queryParams.put("points", "" + current);
+        uri = UriComponentsBuilder.fromHttpUrl("http://127.0.0.1:2001/_dw/rest/carbonj/dbloader/60s7d/foo.bar")
+                .queryParam("points", "{points}")
+                .build(queryParams);
+        try {
+            restTemplate.exchange(uri, HttpMethod.POST, httpEntity, String.class);
+            fail("Should have thrown exception");
+        } catch (HttpServerErrorException e) {
+            assertEquals("500 Server Error: \"{\"status\":\"ERROR\",\"message\":\"Unknown database [60s7d]\"}\"", e.getMessage());
+        }
+
+        uri = UriComponentsBuilder.fromHttpUrl("http://127.0.0.1:2001/_dw/rest/carbonj/dbloader/60s24h/foo.bar")
+                .queryParam("points", "{points}")
+                .build(queryParams);
+        try {
+            restTemplate.exchange(uri, HttpMethod.POST, httpEntity, String.class);
+            fail("Should have thrown exception");
+        } catch (HttpServerErrorException e) {
+            assertEquals("500 Server Error: \"{\"status\":\"ERROR\",\"message\":\"Metric not found. Metric name: [foo.bar]\"}\"", e.getMessage());
+        }
+
+        uri = UriComponentsBuilder.fromHttpUrl("http://127.0.0.1:2001/_dw/rest/carbonj/dbloader/60s24h/a.b.c")
+                .queryParam("points", "{points}")
+                .build(queryParams);
+        try {
+            restTemplate.exchange(uri, HttpMethod.POST, httpEntity, String.class);
+            fail("Should have thrown exception");
+        } catch (HttpServerErrorException e) {
+            assertEquals("500 Server Error: \"{\"status\":\"ERROR\",\"message\":\"Invalid import line format: [" + current + "]\"}\"", e.getMessage());
+        }
+
+        RetentionPolicy retentionPolicy = RetentionPolicy.getInstance("60s:24h");
+        queryParams.put("points", retentionPolicy.interval(current) + ",123.45");
+        queryParams.put("maxImportErrors", "1");
+        uri = UriComponentsBuilder.fromHttpUrl("http://127.0.0.1:2001/_dw/rest/carbonj/dbloader/60s24h/a.b.c")
+                .queryParam("points", "{points}")
+                .queryParam("maxImportErrors", "{maxImportErrors}")
+                .build(queryParams);
+        response = restTemplate.exchange(uri, HttpMethod.POST, httpEntity, String.class);
+        assertEquals(HttpStatus.OK.value(), response.getStatusCode().value());
+        assertEquals("{\"dbName\":\"60s24h\",\"received\":1,\"saved\":1,\"errors\":0,\"expired\":0}", response.getBody());
+    }
+}

carbonj.service/src/test/java/com/demandware/carbonj/service/engine/TestConsumers.java

@@ -19,7 +19,6 @@
 import java.util.List;
 import java.util.Objects;
 
-@Testcontainers
 public class TestConsumers {
 
     @BeforeAll
@@ -43,5 +42,6 @@ public void testConsumers() throws Exception {
         new KinesisRecordProcessorFactory(metricRegistry, new PointProcessorMock(), kinesisConfig, "test-stream");
         consumers.dumpStats();
         consumers.syncNamespaces();
+        consumers.reload();
     }
 }

carbonj.service/src/test/java/com/demandware/carbonj/service/engine/TestDestinationGroup.java

@@ -7,15 +7,28 @@
 package com.demandware.carbonj.service.engine;
 
 import com.codahale.metrics.MetricRegistry;
+import com.demandware.carbonj.service.engine.destination.FileDestination;
+import com.demandware.carbonj.service.engine.destination.KinesisDestination;
 import com.demandware.carbonj.service.engine.destination.LineProtocolDestination;
+import com.demandware.carbonj.service.engine.destination.LineProtocolDestinationSocket;
+import com.demandware.carbonj.service.engine.destination.LineProtocolDestinationsService;
+import com.demandware.carbonj.service.engine.destination.NullDestination;
+import org.apache.commons.io.FileUtils;
 import org.junit.jupiter.api.Test;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.test.annotation.DirtiesContext;
+import software.amazon.awssdk.regions.Region;
 
+import java.io.File;
 import java.util.ArrayList;
 import java.util.List;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT)
 @DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS)
@@ -80,4 +93,34 @@ public void accept(DataPoint dataPoint) {
             dataPoints.add(dataPoint);
         }
     }
+
+    @Test
+    public void test() throws Exception {
+        FileUtils.writeLines(new File("/tmp/kinesis-test-stream-dest.conf"), List.of("kinesisBatchSize=1"));
+        DestinationGroup destinationGroup = new DestinationGroup(metricRegistry, "relay", "",
+                1, 1, 60000, "/tmp", 1, Region.US_EAST_1.id(), false, "", "");
+        assertNull(destinationGroup.getDestinations());
+        destinationGroup = new DestinationGroup(metricRegistry, "relay", "null:foobar",
+                1, 1, 60000, "/tmp", 1, Region.US_EAST_1.id(), false, "", "");
+        assertInstanceOf(NullDestination.class, destinationGroup.getDestinations()[0]);
+        destinationGroup = new DestinationGroup(metricRegistry, "relay", "file:/tmp/destination",
+                1, 1, 60000, "/tmp", 1, Region.US_EAST_1.id(), false, "", "");
+        assertInstanceOf(FileDestination.class, destinationGroup.getDestinations()[0]);
+        destinationGroup = new DestinationGroup(metricRegistry, "relay", "kinesis:test-stream",
+                1, 1, 60000, "/tmp", 1, Region.US_EAST_1.id(), false, "", "");
+        assertInstanceOf(KinesisDestination.class, destinationGroup.getDestinations()[0]);
+        destinationGroup = new DestinationGroup(metricRegistry, "relay", "service:127.0.0.1:8080",
+                1, 1, 60000, "/tmp", 1, Region.US_EAST_1.id(), false, "", "");
+        assertInstanceOf(LineProtocolDestinationsService.class, destinationGroup.getDestinations()[0]);
+        destinationGroup = new DestinationGroup(metricRegistry, "relay", "127.0.0.1:8080",
+                1, 1, 60000, "/tmp", 1, Region.US_EAST_1.id(), false, "", "");
+        assertInstanceOf(LineProtocolDestinationSocket.class, destinationGroup.getDestinations()[0]);
+        assertTrue(destinationGroup.equals(destinationGroup));
+        assertFalse(destinationGroup.equals(new Object()));
+        assertFalse(destinationGroup.equals(new DestinationGroup(metricRegistry, "relay", "127.0.0.1:8088",
+                1, 1, 60000, "/tmp", 1, Region.US_EAST_1.id(), false, "", "")));
+        assertNotEquals(0, destinationGroup.hashCode());
+        assertEquals("127.0.0.1:8080", destinationGroup.getDest());
+        destinationGroup.close();
+    }
 }

carbonj.service/src/test/java/com/demandware/carbonj/service/engine/TestDynamoDbCheckPointMgr.java

@@ -6,30 +6,43 @@
  */
 package com.demandware.carbonj.service.engine;
 
+import com.amazonaws.client.builder.AwsClientBuilder;
 import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
 import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder;
-import org.junit.jupiter.api.Disabled;
+import com.amazonaws.services.dynamodbv2.document.DynamoDB;
 import org.junit.jupiter.api.Test;
-import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.test.annotation.DirtiesContext;
+import org.testcontainers.containers.localstack.LocalStackContainer;
+import org.testcontainers.junit.jupiter.Container;
+import org.testcontainers.junit.jupiter.Testcontainers;
+import org.testcontainers.utility.DockerImageName;
+import software.amazon.awssdk.regions.Region;
 
 import java.util.Date;
 import java.util.concurrent.TimeUnit;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.testcontainers.containers.localstack.LocalStackContainer.Service.DYNAMODB;
 
-@SpringBootTest (webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT)
-@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS)
-@Disabled
+@Testcontainers
 public class TestDynamoDbCheckPointMgr {
 
+    @Container
+    public static LocalStackContainer localstack = new LocalStackContainer(
+            DockerImageName.parse("localstack/localstack:1.4.0")).withServices(DYNAMODB);
+
     @Test
-    @Disabled // ignoring because it is calling AWS API and it should not be
     public void testBasic() throws Exception {
-        AmazonDynamoDB dynamoDbClient = AmazonDynamoDBClientBuilder.standard().build();
+        AmazonDynamoDB dynamoDbClient = AmazonDynamoDBClientBuilder.standard()
+                .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(localstack.getEndpoint().toString(), Region.US_EAST_1.id()))
+                .build();
         CheckPointMgr<Date> checkPointMgr = new DynamoDbCheckPointMgr(dynamoDbClient, "test", 60, 1);
+        while (true) {
+            if (DynamoDbUtils.isTablePresent(new DynamoDB(dynamoDbClient), "checkpoints-test")) break;
+            Thread.sleep(1000);
+        }
         Date lastCheckPoint = checkPointMgr.lastCheckPoint();
+        Thread.sleep(100);
         assertTrue(lastCheckPoint.before(new Date(System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(60))));
         Date checkPoint1 = new Date(System.currentTimeMillis());
         checkPointMgr.checkPoint(checkPoint1);

carbonj.service/src/test/java/com/demandware/carbonj/service/engine/TestGraphiteMetricSearchServlet.java

@@ -0,0 +1,58 @@
+/**
+ * Copyright (c) 2018, salesforce.com, inc.
+ * All rights reserved.
+ * SPDX-License-Identifier: BSD-3-Clause
+ * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+ */
+package com.demandware.carbonj.service.engine;
+
+import com.demandware.carbonj.service.db.model.MsgPackMetric;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.joda.time.DateTime;
+import org.junit.jupiter.api.Test;
+import org.msgpack.jackson.dataformat.MessagePackMapper;
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpMethod;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.client.RestTemplate;
+import org.springframework.web.util.UriComponentsBuilder;
+
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class TestGraphiteMetricSearchServlet extends CarbonJSvcLongIdTest {
+    @Test
+    public void testGraphiteMetricSearchServlet() throws Exception {
+        DateTime now = DateTime.now();
+        cjClient.send( "a.b.c", 1.0f, now );
+        cjClient.send( "a.b.d", 1.0f, now );
+        drain();
+
+        RestTemplate restTemplate = new RestTemplate();
+        Map<String, String> queryParams = new HashMap<>();
+        queryParams.put("format", "msgpack");
+        queryParams.put("query", "a.b.c");
+        URI uri = UriComponentsBuilder.fromHttpUrl("http://127.0.0.1:2001/metrics")
+                .queryParam("format", "{format}")
+                .queryParam("query", "{query}")
+                .build(queryParams);
+        HttpEntity<String> httpEntity = new HttpEntity<>(new HttpHeaders());
+        ResponseEntity<String> response = restTemplate.exchange(uri, HttpMethod.GET, httpEntity, String.class);
+        assertEquals(HttpStatus.OK.value(), response.getStatusCode().value());
+        ObjectMapper objectMapper = new MessagePackMapper();
+        List<MsgPackMetric> msgPackMetricList = objectMapper.readValue(response.getBody().getBytes(StandardCharsets.ISO_8859_1), new TypeReference<>() {});
+        assertEquals(1, msgPackMetricList.size());
+        MsgPackMetric msgPackMetric = msgPackMetricList.get(0);
+        assertEquals("a.b.c", msgPackMetric.path);
+        assertTrue(msgPackMetric.isLeaf);
+    }
+}