From d2ea2324a146e2c4e595ceafad095c6a868b6435 Mon Sep 17 00:00:00 2001 From: Rasmus Jokinen <146736881+51-code@users.noreply.github.com> Date: Tue, 24 Sep 2024 16:05:21 +0300 Subject: [PATCH 01/10] Allow splitting incoming messages with JSON array format Delete PayloadConfig and include configuration in ConversionFactory that provides the correct object instead Add secondary ctor to RegexConversion Use Collections.singletonList in DefaultPayload, RegexConversion as final class Add object equality test for RegexPayload Move tests to the correct package Move config tests from RegexPayloadTest to PayloadConfigTest and add tests for splitType Refactor split() function in Payload Refactor JsonPayload and RegexPayload to encapsulate Payload instead of String Rename Payload's take() function to message() Implement json_array splitting Refactor Payload objects and regex splitting Remove duplicate tests from RegexSplittingTest Apply spotless Move regex splitting tests to their own file, refactor the tests Change config to allow splitting with json_array Refactor regex splitting to a decorator --- README.adoc | 4 +- etc/config.properties | 4 +- pom.xml | 11 + .../com/teragrep/lsh_01/HttpInitializer.java | 1 + .../teragrep/lsh_01/HttpServerHandler.java | 1 + src/main/java/com/teragrep/lsh_01/Main.java | 28 ++- .../com/teragrep/lsh_01/MessageProcessor.java | 1 + .../com/teragrep/lsh_01/NettyHttpServer.java | 1 + .../teragrep/lsh_01/config/Configuration.java | 34 +++ .../lsh_01/config/ConversionFactory.java | 130 +++++++++++ .../lsh_01/config/PathProperties.java | 54 +++++ .../teragrep/lsh_01/config/PayloadConfig.java | 56 ----- .../lsh_01/conversion/DefaultPayload.java | 49 +++++ .../{ => conversion}/IMessageHandler.java | 2 +- .../lsh_01/conversion/JsonConversion.java | 70 ++++++ .../lsh_01/conversion/JsonPayload.java | 70 ++++++ .../teragrep/lsh_01/conversion/Payload.java | 35 +++ .../lsh_01/conversion/RegexConversion.java | 77 +++++++ .../RegexPayload.java} | 45 ++-- .../{ => conversion}/RelpConversion.java | 29 +-- .../lsh_01/ConversionFactoryTest.java | 146 +++++++++++++ .../com/teragrep/lsh_01/CredentialsTest.java | 32 +-- .../com/teragrep/lsh_01/EndToEndTest.java | 77 ++----- .../com/teragrep/lsh_01/JsonPayloadTest.java | 76 +++++++ .../teragrep/lsh_01/JsonSplittingTest.java | 205 ++++++++++++++++++ .../java/com/teragrep/lsh_01/LookupTest.java | 11 +- .../java/com/teragrep/lsh_01/PayloadTest.java | 108 --------- .../com/teragrep/lsh_01/RegexPayloadTest.java | 87 ++++++++ .../teragrep/lsh_01/RegexSplittingTest.java | 126 +++++++++++ 29 files changed, 1257 insertions(+), 313 deletions(-) create mode 100644 src/main/java/com/teragrep/lsh_01/config/Configuration.java create mode 100644 src/main/java/com/teragrep/lsh_01/config/ConversionFactory.java create mode 100644 src/main/java/com/teragrep/lsh_01/config/PathProperties.java delete mode 100644 src/main/java/com/teragrep/lsh_01/config/PayloadConfig.java create mode 100644 src/main/java/com/teragrep/lsh_01/conversion/DefaultPayload.java rename src/main/java/com/teragrep/lsh_01/{ => conversion}/IMessageHandler.java (97%) create mode 100644 src/main/java/com/teragrep/lsh_01/conversion/JsonConversion.java create mode 100644 src/main/java/com/teragrep/lsh_01/conversion/JsonPayload.java create mode 100644 src/main/java/com/teragrep/lsh_01/conversion/Payload.java create mode 100644 src/main/java/com/teragrep/lsh_01/conversion/RegexConversion.java rename src/main/java/com/teragrep/lsh_01/{Payload.java => conversion/RegexPayload.java} (51%) rename src/main/java/com/teragrep/lsh_01/{ => conversion}/RelpConversion.java (80%) create mode 100644 src/test/java/com/teragrep/lsh_01/ConversionFactoryTest.java create mode 100644 src/test/java/com/teragrep/lsh_01/JsonPayloadTest.java create mode 100644 src/test/java/com/teragrep/lsh_01/JsonSplittingTest.java create mode 100644 src/test/java/com/teragrep/lsh_01/RegexPayloadTest.java create mode 100644 src/test/java/com/teragrep/lsh_01/RegexSplittingTest.java diff --git a/README.adoc b/README.adoc index 710aa0c..ca8f517 100644 --- a/README.adoc +++ b/README.adoc @@ -35,8 +35,8 @@ security.authRequired,true,Sets whether Basic HTTP Authorization headers are req credentials.file,etc/credentials.json,A json file with array of identity:credential mappings lookups.hostname.file,etc/hostname.json,Path to username-to-hostname lookup table lookups.appname.file,etc/appname.json,Path to username-to-appname lookup table -payload.splitRegex, \n (newline), A regex based on which incoming requests will be split into multiple outgoing messages -payload.splitEnabled, false, Sets whether splitting incoming messages by splitRegex is enabled +payload.splitType, none, Sets how to split incoming messages. Supported split types are 'regex' and 'json_array'. Use 'none' for no splitting. +payload.splitType.regex.pattern, \n (newline), A regex based on which incoming requests will be split into multiple outgoing messages prometheus.port, 1234, Port used by the server that provides DropWizard metrics |=== diff --git a/etc/config.properties b/etc/config.properties index 1e5ea81..2809003 100644 --- a/etc/config.properties +++ b/etc/config.properties @@ -20,7 +20,7 @@ credentials.file=etc/credentials.json lookups.hostname.file=etc/hostname.json lookups.appname.file=etc/appname.json -payload.splitRegex=\n -payload.splitEnabled=false +payload.splitType=none +payload.splitType.regex.pattern=\n prometheus.port=1234 diff --git a/pom.xml b/pom.xml index 36bf106..a95d535 100644 --- a/pom.xml +++ b/pom.xml @@ -140,6 +140,17 @@ log4j-core ${log4j.version} + + + jakarta.json + jakarta.json-api + 2.1.3 + + + org.eclipse.parsson + parsson + 1.1.7 + org.junit.jupiter diff --git a/src/main/java/com/teragrep/lsh_01/HttpInitializer.java b/src/main/java/com/teragrep/lsh_01/HttpInitializer.java index 07e3e76..1bc2973 100644 --- a/src/main/java/com/teragrep/lsh_01/HttpInitializer.java +++ b/src/main/java/com/teragrep/lsh_01/HttpInitializer.java @@ -20,6 +20,7 @@ package com.teragrep.lsh_01; import com.teragrep.lsh_01.config.InternalEndpointUrlConfig; +import com.teragrep.lsh_01.conversion.IMessageHandler; import com.teragrep.lsh_01.util.LoggingHttpObjectAggregator; import com.teragrep.lsh_01.util.SslHandlerProvider; import io.netty.channel.ChannelInitializer; diff --git a/src/main/java/com/teragrep/lsh_01/HttpServerHandler.java b/src/main/java/com/teragrep/lsh_01/HttpServerHandler.java index 3da8f50..e54a8db 100644 --- a/src/main/java/com/teragrep/lsh_01/HttpServerHandler.java +++ b/src/main/java/com/teragrep/lsh_01/HttpServerHandler.java @@ -20,6 +20,7 @@ package com.teragrep.lsh_01; import com.teragrep.lsh_01.config.InternalEndpointUrlConfig; +import com.teragrep.lsh_01.conversion.IMessageHandler; import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.SimpleChannelInboundHandler; diff --git a/src/main/java/com/teragrep/lsh_01/Main.java b/src/main/java/com/teragrep/lsh_01/Main.java index 9249639..eb3126d 100644 --- a/src/main/java/com/teragrep/lsh_01/Main.java +++ b/src/main/java/com/teragrep/lsh_01/Main.java @@ -27,24 +27,34 @@ import com.teragrep.lsh_01.metrics.JmxReport; import com.teragrep.lsh_01.metrics.Report; import com.teragrep.lsh_01.metrics.Slf4jReport; +import com.teragrep.lsh_01.conversion.*; import com.teragrep.lsh_01.pool.*; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.io.IOException; +import java.util.Map; public class Main { private final static Logger LOGGER = LogManager.getLogger(Main.class); public static void main(String[] args) { + Map propsMap; + try { + propsMap = new PathProperties(System.getProperty("properties.file", "etc/config.properties")) + .deepCopyAsUnmodifiableMap(); + } + catch (IOException e) { + throw new IllegalArgumentException("Can't find properties file: ", e); + } + NettyConfig nettyConfig = new NettyConfig(); RelpConfig relpConfig = new RelpConfig(); SecurityConfig securityConfig = new SecurityConfig(); BasicAuthentication basicAuthentication = new BasicAuthenticationFactory().create(); InternalEndpointUrlConfig internalEndpointUrlConfig = new InternalEndpointUrlConfig(); LookupConfig lookupConfig = new LookupConfig(); - PayloadConfig payloadConfig = new PayloadConfig(); MetricsConfig metricsConfig = new MetricsConfig(); try { nettyConfig.validate(); @@ -52,7 +62,6 @@ public static void main(String[] args) { securityConfig.validate(); internalEndpointUrlConfig.validate(); lookupConfig.validate(); - payloadConfig.validate(); metricsConfig.validate(); } catch (IllegalArgumentException e) { @@ -63,7 +72,6 @@ public static void main(String[] args) { LOGGER.info("Got relp config: <[{}]>", relpConfig); LOGGER.info("Got internal endpoint config: <[{}]>", internalEndpointUrlConfig); LOGGER.info("Got lookup table config: <[{}]>", lookupConfig); - LOGGER.info("Got payload config: <[{}]>", payloadConfig); LOGGER.info("Authentication required: <[{}]>", securityConfig.authRequired); // metrics @@ -76,14 +84,18 @@ public static void main(String[] args) { RelpConnectionFactory relpConnectionFactory = new RelpConnectionFactory(relpConfig, metricRegistry); Pool pool = new Pool<>(relpConnectionFactory, new ManagedRelpConnectionStub()); - IMessageHandler relpConversion = new MetricRelpConversion( - new RelpConversion(pool, securityConfig, basicAuthentication, lookupConfig, payloadConfig), - metricRegistry - ); + IMessageHandler conversion = new MetricRelpConversion(new ConversionFactory( + propsMap, + pool, + securityConfig, + basicAuthentication, + lookupConfig + ).conversion(), + metricRegistry); try ( HttpServer server = new MetricHttpServer( - new NettyHttpServer(nettyConfig, relpConversion, null, 200, internalEndpointUrlConfig), + new NettyHttpServer(nettyConfig, conversion, null, 200, internalEndpointUrlConfig), report ) ) { diff --git a/src/main/java/com/teragrep/lsh_01/MessageProcessor.java b/src/main/java/com/teragrep/lsh_01/MessageProcessor.java index 8476dd1..dfd240f 100644 --- a/src/main/java/com/teragrep/lsh_01/MessageProcessor.java +++ b/src/main/java/com/teragrep/lsh_01/MessageProcessor.java @@ -21,6 +21,7 @@ import com.teragrep.lsh_01.authentication.*; import com.teragrep.lsh_01.config.InternalEndpointUrlConfig; +import com.teragrep.lsh_01.conversion.IMessageHandler; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import io.netty.channel.ChannelHandlerContext; diff --git a/src/main/java/com/teragrep/lsh_01/NettyHttpServer.java b/src/main/java/com/teragrep/lsh_01/NettyHttpServer.java index 353bc95..07c2c0a 100644 --- a/src/main/java/com/teragrep/lsh_01/NettyHttpServer.java +++ b/src/main/java/com/teragrep/lsh_01/NettyHttpServer.java @@ -21,6 +21,7 @@ import com.teragrep.lsh_01.config.InternalEndpointUrlConfig; import com.teragrep.lsh_01.config.NettyConfig; +import com.teragrep.lsh_01.conversion.IMessageHandler; import io.netty.bootstrap.ServerBootstrap; import io.netty.channel.ChannelFuture; import io.netty.channel.ChannelOption; diff --git a/src/main/java/com/teragrep/lsh_01/config/Configuration.java b/src/main/java/com/teragrep/lsh_01/config/Configuration.java new file mode 100644 index 0000000..b66515f --- /dev/null +++ b/src/main/java/com/teragrep/lsh_01/config/Configuration.java @@ -0,0 +1,34 @@ +/* + logstash-http-input to syslog bridge + Copyright 2024 Suomen Kanuuna Oy + + Derivative Work of Elasticsearch + Copyright 2012-2015 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.teragrep.lsh_01.config; + +import java.io.IOException; +import java.util.Map; + +public interface Configuration { + + /** + * Get configuration as an unmodifiable map so that it can't be altered anymore. + * + * @return configuration as an unmodifiable map + * @throws IOException if configuration file is not found + */ + Map deepCopyAsUnmodifiableMap() throws IOException; +} diff --git a/src/main/java/com/teragrep/lsh_01/config/ConversionFactory.java b/src/main/java/com/teragrep/lsh_01/config/ConversionFactory.java new file mode 100644 index 0000000..0a6bde9 --- /dev/null +++ b/src/main/java/com/teragrep/lsh_01/config/ConversionFactory.java @@ -0,0 +1,130 @@ +/* + logstash-http-input to syslog bridge + Copyright 2024 Suomen Kanuuna Oy + + Derivative Work of Elasticsearch + Copyright 2012-2015 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.teragrep.lsh_01.config; + +import com.teragrep.lsh_01.authentication.BasicAuthentication; +import com.teragrep.lsh_01.conversion.IMessageHandler; +import com.teragrep.lsh_01.conversion.JsonConversion; +import com.teragrep.lsh_01.conversion.RegexConversion; +import com.teragrep.lsh_01.conversion.RelpConversion; +import com.teragrep.lsh_01.pool.IManagedRelpConnection; +import com.teragrep.lsh_01.pool.Pool; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.Map; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +public final class ConversionFactory { + + private final static Logger LOGGER = LogManager.getLogger(ConversionFactory.class); + + private final String splitType; + private final String regexPattern; + private final Pool pool; + private final SecurityConfig securityConfig; + private final BasicAuthentication basicAuthentication; + private final LookupConfig lookupConfig; + + public ConversionFactory( + Map configuration, + Pool pool, + SecurityConfig securityConfig, + BasicAuthentication basicAuthentication, + LookupConfig lookupConfig + ) { + // if system property is not specified, defaults to config file (the Map) + this( + System.getProperty("payload.splitType", configuration.get("payload.splitType")), + System + .getProperty( + "payload.splitType.regex.pattern", configuration.get("payload.splitType.regex.pattern") + ), + pool, + securityConfig, + basicAuthentication, + lookupConfig + ); + } + + public ConversionFactory( + String splitType, + String regexPattern, + Pool pool, + SecurityConfig securityConfig, + BasicAuthentication basicAuthentication, + LookupConfig lookupConfig + ) { + this.splitType = splitType; + this.regexPattern = regexPattern; + this.pool = pool; + this.securityConfig = securityConfig; + this.basicAuthentication = basicAuthentication; + this.lookupConfig = lookupConfig; + } + + public IMessageHandler conversion() { + LOGGER + .info( + "Creating IMessageHandler with given configuration: payload.splitType=<[{}]>, payload.splitType.regex.pattern=<[{}]>", + splitType, regexPattern + ); + + validateConfiguration(); + + IMessageHandler conversion = new RelpConversion(pool, securityConfig, basicAuthentication, lookupConfig); + + // apply splitting if configured. "none" value is skipped + switch (splitType) { + case "regex": + conversion = new RegexConversion(conversion, regexPattern); + break; + case "json_array": + conversion = new JsonConversion(conversion); + break; + } + + return conversion; + } + + private void validateConfiguration() { + switch (splitType) { + case "regex": + try { + Pattern.compile(regexPattern); + } + catch (PatternSyntaxException e) { + throw new IllegalArgumentException( + "Configuration has an invalid regex (payload.splitType.regex.pattern): " + regexPattern + ); + } + break; + case "json_array": + case "none": + break; + default: + throw new IllegalArgumentException( + "Configuration has an invalid splitType: " + splitType + + ". Has to be 'regex', 'json_array' or 'none'." + ); + } + } +} diff --git a/src/main/java/com/teragrep/lsh_01/config/PathProperties.java b/src/main/java/com/teragrep/lsh_01/config/PathProperties.java new file mode 100644 index 0000000..727f83b --- /dev/null +++ b/src/main/java/com/teragrep/lsh_01/config/PathProperties.java @@ -0,0 +1,54 @@ +/* + logstash-http-input to syslog bridge + Copyright 2024 Suomen Kanuuna Oy + + Derivative Work of Elasticsearch + Copyright 2012-2015 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.teragrep.lsh_01.config; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.Collections; +import java.util.Map; +import java.util.Properties; +import java.util.stream.Collectors; + +public final class PathProperties implements Configuration { + + private final File file; + + public PathProperties(String fileName) { + this.file = new File(fileName); + } + + public PathProperties(File file) { + this.file = file; + } + + @Override + public Map deepCopyAsUnmodifiableMap() throws IOException { + Properties properties = new Properties(); + properties.load(new FileInputStream(file)); + return Collections + .unmodifiableMap( + properties + .entrySet() + .stream() + .collect(Collectors.toMap(k -> k.getKey().toString(), k -> k.getValue().toString())) + ); + } +} diff --git a/src/main/java/com/teragrep/lsh_01/config/PayloadConfig.java b/src/main/java/com/teragrep/lsh_01/config/PayloadConfig.java deleted file mode 100644 index dc2260a..0000000 --- a/src/main/java/com/teragrep/lsh_01/config/PayloadConfig.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - logstash-http-input to syslog bridge - Copyright 2024 Suomen Kanuuna Oy - - Derivative Work of Elasticsearch - Copyright 2012-2015 Elasticsearch - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ -package com.teragrep.lsh_01.config; - -import java.util.regex.Pattern; -import java.util.regex.PatternSyntaxException; - -public class PayloadConfig implements Validateable { - - public final String splitRegex; - public final boolean splitEnabled; - - public PayloadConfig() { - PropertiesReaderUtilityClass propertiesReader = new PropertiesReaderUtilityClass( - System.getProperty("properties.file", "etc/config.properties") - ); - splitRegex = propertiesReader.getStringProperty("payload.splitRegex"); - splitEnabled = propertiesReader.getBooleanProperty("payload.splitEnabled"); - } - - @Override - public void validate() { - if (splitEnabled) { - try { - Pattern.compile(splitRegex); - } - catch (PatternSyntaxException e) { - throw new IllegalArgumentException( - "Configuration has an invalid regex (payload.splitRegex): " + splitRegex - ); - } - } - } - - @Override - public String toString() { - return "PayloadConfig{" + "splitRegex=" + splitRegex + ", splitEnabled=" + splitEnabled + '}'; - } -} diff --git a/src/main/java/com/teragrep/lsh_01/conversion/DefaultPayload.java b/src/main/java/com/teragrep/lsh_01/conversion/DefaultPayload.java new file mode 100644 index 0000000..3065267 --- /dev/null +++ b/src/main/java/com/teragrep/lsh_01/conversion/DefaultPayload.java @@ -0,0 +1,49 @@ +/* + logstash-http-input to syslog bridge + Copyright 2024 Suomen Kanuuna Oy + + Derivative Work of Elasticsearch + Copyright 2012-2015 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.teragrep.lsh_01.conversion; + +import java.util.Collections; +import java.util.List; + +public final class DefaultPayload implements Payload { + + private final String message; + + public DefaultPayload(String message) { + this.message = message; + } + + @Override + public List messages() { + return Collections.singletonList(message); + } + + @Override + public boolean equals(final Object object) { + if (this == object) + return true; + if (object == null) + return false; + if (object.getClass() != this.getClass()) + return false; + final DefaultPayload cast = (DefaultPayload) object; + return message.equals(cast.message); + } +} diff --git a/src/main/java/com/teragrep/lsh_01/IMessageHandler.java b/src/main/java/com/teragrep/lsh_01/conversion/IMessageHandler.java similarity index 97% rename from src/main/java/com/teragrep/lsh_01/IMessageHandler.java rename to src/main/java/com/teragrep/lsh_01/conversion/IMessageHandler.java index 8b4b4cf..cc51ba5 100644 --- a/src/main/java/com/teragrep/lsh_01/IMessageHandler.java +++ b/src/main/java/com/teragrep/lsh_01/conversion/IMessageHandler.java @@ -17,7 +17,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package com.teragrep.lsh_01; +package com.teragrep.lsh_01.conversion; import com.teragrep.lsh_01.authentication.Subject; diff --git a/src/main/java/com/teragrep/lsh_01/conversion/JsonConversion.java b/src/main/java/com/teragrep/lsh_01/conversion/JsonConversion.java new file mode 100644 index 0000000..0df2ca2 --- /dev/null +++ b/src/main/java/com/teragrep/lsh_01/conversion/JsonConversion.java @@ -0,0 +1,70 @@ +/* + logstash-http-input to syslog bridge + Copyright 2024 Suomen Kanuuna Oy + + Derivative Work of Elasticsearch + Copyright 2012-2015 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.teragrep.lsh_01.conversion; + +import com.teragrep.lsh_01.authentication.Subject; + +import java.util.Map; + +/** + * Decorator for IMessageHandler that splits messages arriving as an array of JSON objects. + */ +public final class JsonConversion implements IMessageHandler { + + private final IMessageHandler conversion; + + public JsonConversion(IMessageHandler conversion) { + this.conversion = conversion; + } + + @Override + public boolean onNewMessage(Subject subject, Map headers, String body) { + JsonPayload originalPayload = new JsonPayload(new DefaultPayload(body)); + + boolean msgSent = true; + for (String message : originalPayload.messages()) { // each object individually as a String + if (!conversion.onNewMessage(subject, headers, message)) { + msgSent = false; + } + } + + return msgSent; + } + + @Override + public Subject asSubject(String token) { + return conversion.asSubject(token); + } + + @Override + public boolean requiresToken() { + return conversion.requiresToken(); + } + + @Override + public IMessageHandler copy() { + return new JsonConversion(conversion.copy()); + } + + @Override + public Map responseHeaders() { + return conversion.responseHeaders(); + } +} diff --git a/src/main/java/com/teragrep/lsh_01/conversion/JsonPayload.java b/src/main/java/com/teragrep/lsh_01/conversion/JsonPayload.java new file mode 100644 index 0000000..18b0c56 --- /dev/null +++ b/src/main/java/com/teragrep/lsh_01/conversion/JsonPayload.java @@ -0,0 +1,70 @@ +/* + logstash-http-input to syslog bridge + Copyright 2024 Suomen Kanuuna Oy + + Derivative Work of Elasticsearch + Copyright 2012-2015 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.teragrep.lsh_01.conversion; + +import jakarta.json.*; + +import java.io.StringReader; +import java.util.ArrayList; +import java.util.List; + +/** + * A Json array payload splittable into individual json objects. + */ +public final class JsonPayload implements Payload { + + private final Payload payload; + + public JsonPayload(Payload payload) { + this.payload = payload; + } + + /** + * Splits the array of JSON objects into payloads with one object each. Has a side effect of removing whitespace + * from the payloads because of jsonObject.toString(). + * + * @return list of messages + */ + @Override + public List messages() { + List allMessages = new ArrayList<>(); + for (String message : payload.messages()) { + JsonReader reader = Json.createReader(new StringReader(message)); + JsonArray payloadMessages = reader.readArray(); + + // transform all json objects into DefaultPayloads and return the list + allMessages.addAll(payloadMessages.getValuesAs(JsonObject::toString)); + } + + return allMessages; + } + + @Override + public boolean equals(final Object object) { + if (this == object) + return true; + if (object == null) + return false; + if (object.getClass() != this.getClass()) + return false; + final JsonPayload cast = (JsonPayload) object; + return payload.equals(cast.payload); + } +} diff --git a/src/main/java/com/teragrep/lsh_01/conversion/Payload.java b/src/main/java/com/teragrep/lsh_01/conversion/Payload.java new file mode 100644 index 0000000..554f244 --- /dev/null +++ b/src/main/java/com/teragrep/lsh_01/conversion/Payload.java @@ -0,0 +1,35 @@ +/* + logstash-http-input to syslog bridge + Copyright 2024 Suomen Kanuuna Oy + + Derivative Work of Elasticsearch + Copyright 2012-2015 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.teragrep.lsh_01.conversion; + +import java.util.List; + +/** + * A message from a log source + */ +public interface Payload { + + /** + * Takes the message(s) from the payload. + * + * @return list of messages + */ + List messages(); +} diff --git a/src/main/java/com/teragrep/lsh_01/conversion/RegexConversion.java b/src/main/java/com/teragrep/lsh_01/conversion/RegexConversion.java new file mode 100644 index 0000000..9c32ffb --- /dev/null +++ b/src/main/java/com/teragrep/lsh_01/conversion/RegexConversion.java @@ -0,0 +1,77 @@ +/* + logstash-http-input to syslog bridge + Copyright 2024 Suomen Kanuuna Oy + + Derivative Work of Elasticsearch + Copyright 2012-2015 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.teragrep.lsh_01.conversion; + +import com.teragrep.lsh_01.authentication.Subject; + +import java.util.Map; +import java.util.regex.Pattern; + +/** + * Decorator for IMessageHandler that splits messages based on given regex. + */ +public final class RegexConversion implements IMessageHandler { + + private final IMessageHandler conversion; + private final Pattern pattern; + + public RegexConversion(IMessageHandler conversion, String regex) { + this(conversion, Pattern.compile(regex)); + } + + public RegexConversion(IMessageHandler conversion, Pattern pattern) { + this.conversion = conversion; + this.pattern = pattern; + } + + @Override + public boolean onNewMessage(Subject subject, Map headers, String body) { + RegexPayload originalPayload = new RegexPayload(new DefaultPayload(body), pattern); + + boolean msgSent = true; + for (String message : originalPayload.messages()) { // process each split message individually + if (!conversion.onNewMessage(subject, headers, message)) { + msgSent = false; + } + } + + return msgSent; + } + + @Override + public Subject asSubject(String token) { + return conversion.asSubject(token); + } + + @Override + public boolean requiresToken() { + return conversion.requiresToken(); + } + + @Override + public IMessageHandler copy() { + return new RegexConversion(conversion.copy(), pattern); + } + + @Override + public Map responseHeaders() { + return conversion.responseHeaders(); + } +} diff --git a/src/main/java/com/teragrep/lsh_01/Payload.java b/src/main/java/com/teragrep/lsh_01/conversion/RegexPayload.java similarity index 51% rename from src/main/java/com/teragrep/lsh_01/Payload.java rename to src/main/java/com/teragrep/lsh_01/conversion/RegexPayload.java index 73b079b..64abb15 100644 --- a/src/main/java/com/teragrep/lsh_01/Payload.java +++ b/src/main/java/com/teragrep/lsh_01/conversion/RegexPayload.java @@ -17,48 +17,51 @@ See the License for the specific language governing permissions and limitations under the License. */ -package com.teragrep.lsh_01; +package com.teragrep.lsh_01.conversion; import java.util.ArrayList; import java.util.List; import java.util.regex.Pattern; /** - * A message from a log source + * Payload splittable with a regex pattern. */ -final public class Payload { +final public class RegexPayload implements Payload { - private final String body; + private final Payload payload; private final Pattern splitPattern; - public Payload(String body, Pattern splitPattern) { - this.body = body; + public RegexPayload(Payload payload, Pattern splitPattern) { + this.payload = payload; this.splitPattern = splitPattern; } /** * Splits the payload into multiple payloads if there is a defined split regex in the body. - * + * * @return list of Payloads */ - public List split() { - ArrayList payloads = new ArrayList<>(); + @Override + public List messages() { + ArrayList allMessages = new ArrayList<>(); - String[] messages = splitPattern.split(body); - - for (String message : messages) { - payloads.add(new Payload(message, splitPattern)); + for (String message : payload.messages()) { + String[] payloadMessages = splitPattern.split(message); + allMessages.addAll(List.of(payloadMessages)); } - return payloads; + return allMessages; } - /** - * Takes the message from the payload. - * - * @return message body - */ - public String take() { - return body; + @Override + public boolean equals(final Object object) { + if (this == object) + return true; + if (object == null) + return false; + if (object.getClass() != this.getClass()) + return false; + final RegexPayload cast = (RegexPayload) object; + return payload.equals(cast.payload) && splitPattern.equals(cast.splitPattern); } } diff --git a/src/main/java/com/teragrep/lsh_01/RelpConversion.java b/src/main/java/com/teragrep/lsh_01/conversion/RelpConversion.java similarity index 80% rename from src/main/java/com/teragrep/lsh_01/RelpConversion.java rename to src/main/java/com/teragrep/lsh_01/conversion/RelpConversion.java index e1bccf2..1303a65 100644 --- a/src/main/java/com/teragrep/lsh_01/RelpConversion.java +++ b/src/main/java/com/teragrep/lsh_01/conversion/RelpConversion.java @@ -17,13 +17,12 @@ See the License for the specific language governing permissions and limitations under the License. */ -package com.teragrep.lsh_01; +package com.teragrep.lsh_01.conversion; import com.teragrep.jlt_01.StringLookupTable; import com.teragrep.lsh_01.authentication.BasicAuthentication; import com.teragrep.lsh_01.authentication.Subject; import com.teragrep.lsh_01.config.LookupConfig; -import com.teragrep.lsh_01.config.PayloadConfig; import com.teragrep.lsh_01.config.SecurityConfig; import com.teragrep.lsh_01.lookup.LookupTableFactory; import com.teragrep.lsh_01.pool.*; @@ -35,7 +34,6 @@ import java.time.Instant; import java.util.HashMap; import java.util.Map; -import java.util.regex.Pattern; public class RelpConversion implements IMessageHandler { @@ -44,7 +42,6 @@ public class RelpConversion implements IMessageHandler { private final SecurityConfig securityConfig; private final BasicAuthentication basicAuthentication; private final LookupConfig lookupConfig; - private final PayloadConfig payloadConfig; private final StringLookupTable hostnameLookup; private final StringLookupTable appnameLookup; @@ -52,8 +49,7 @@ public RelpConversion( Pool relpConnectionPool, SecurityConfig securityConfig, BasicAuthentication basicAuthentication, - LookupConfig lookupConfig, - PayloadConfig payloadConfig + LookupConfig lookupConfig ) { this.relpConnectionPool = relpConnectionPool; this.securityConfig = securityConfig; @@ -61,26 +57,13 @@ public RelpConversion( this.lookupConfig = lookupConfig; this.hostnameLookup = new LookupTableFactory().create(lookupConfig.hostnamePath); this.appnameLookup = new LookupTableFactory().create(lookupConfig.appNamePath); - this.payloadConfig = payloadConfig; } public boolean onNewMessage(Subject subject, Map headers, String body) { try { - if (payloadConfig.splitEnabled) { - Pattern splitPattern = Pattern.compile(payloadConfig.splitRegex); - Payload originalPayload = new Payload(body, splitPattern); - - for (Payload payload : originalPayload.split()) { - sendMessage( - payload.take(), headers, subject.subject(), hostnameLookup.lookup(subject.subject()), appnameLookup.lookup(subject.subject()) - ); - } - } - else { - sendMessage( - body, headers, subject.subject(), hostnameLookup.lookup(subject.subject()), appnameLookup.lookup(subject.subject()) - ); - } + sendMessage( + body, headers, subject.subject(), hostnameLookup.lookup(subject.subject()), appnameLookup.lookup(subject.subject()) + ); } catch (Exception e) { LOGGER.error("Unexpected error when sending a message: <{}>", e.getMessage(), e); @@ -99,7 +82,7 @@ public boolean requiresToken() { public RelpConversion copy() { LOGGER.debug("RelpConversion.copy called"); - return new RelpConversion(relpConnectionPool, securityConfig, basicAuthentication, lookupConfig, payloadConfig); + return new RelpConversion(relpConnectionPool, securityConfig, basicAuthentication, lookupConfig); } public Map responseHeaders() { diff --git a/src/test/java/com/teragrep/lsh_01/ConversionFactoryTest.java b/src/test/java/com/teragrep/lsh_01/ConversionFactoryTest.java new file mode 100644 index 0000000..eade9b3 --- /dev/null +++ b/src/test/java/com/teragrep/lsh_01/ConversionFactoryTest.java @@ -0,0 +1,146 @@ +/* + logstash-http-input to syslog bridge + Copyright 2024 Suomen Kanuuna Oy + + Derivative Work of Elasticsearch + Copyright 2012-2015 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.teragrep.lsh_01; + +import com.teragrep.lsh_01.authentication.BasicAuthenticationFactory; +import com.teragrep.lsh_01.config.*; +import com.teragrep.lsh_01.pool.IManagedRelpConnection; +import com.teragrep.lsh_01.pool.ManagedRelpConnectionStub; +import com.teragrep.lsh_01.pool.Pool; +import com.teragrep.lsh_01.pool.RelpConnectionFactory; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class ConversionFactoryTest { + + @Test + public void testInvalidSplitRegex() { + String regexPattern = "(a*b{)"; + String splitType = "regex"; + + RelpConnectionFactory relpConnectionFactory = new RelpConnectionFactory(new RelpConfig()); + Pool pool = new Pool<>(relpConnectionFactory, new ManagedRelpConnectionStub()); + + ConversionFactory conversionFactory = new ConversionFactory( + splitType, + regexPattern, + pool, + new SecurityConfig(), + new BasicAuthenticationFactory().create(), + new LookupConfig() + ); + IllegalArgumentException e = Assertions + .assertThrows(IllegalArgumentException.class, conversionFactory::conversion); + + Assertions + .assertEquals( + "Configuration has an invalid regex (payload.splitType.regex.pattern): " + regexPattern, + e.getMessage() + ); + } + + @Test + public void testValidSplitRegex() { + String regexPattern = "\\n"; + String splitType = "regex"; + + RelpConnectionFactory relpConnectionFactory = new RelpConnectionFactory(new RelpConfig()); + Pool pool = new Pool<>(relpConnectionFactory, new ManagedRelpConnectionStub()); + + ConversionFactory conversionFactory = new ConversionFactory( + splitType, + regexPattern, + pool, + new SecurityConfig(), + new BasicAuthenticationFactory().create(), + new LookupConfig() + ); + + Assertions.assertDoesNotThrow(conversionFactory::conversion); + } + + @Test + public void testInvalidSplitType() { + String splitType = "invalid"; + String regexPattern = ""; + + RelpConnectionFactory relpConnectionFactory = new RelpConnectionFactory(new RelpConfig()); + Pool pool = new Pool<>(relpConnectionFactory, new ManagedRelpConnectionStub()); + + ConversionFactory conversionFactory = new ConversionFactory( + splitType, + regexPattern, + pool, + new SecurityConfig(), + new BasicAuthenticationFactory().create(), + new LookupConfig() + ); + + IllegalArgumentException e = Assertions + .assertThrows(IllegalArgumentException.class, conversionFactory::conversion); + Assertions + .assertEquals( + "Configuration has an invalid splitType: " + splitType + + ". Has to be 'regex', 'json_array' or 'none'.", + e.getMessage() + ); + } + + @Test + public void testValidSplitType() { + String regexPattern = ""; + String splitType = "json_array"; + + RelpConnectionFactory relpConnectionFactory = new RelpConnectionFactory(new RelpConfig()); + Pool pool = new Pool<>(relpConnectionFactory, new ManagedRelpConnectionStub()); + + ConversionFactory conversionFactory = new ConversionFactory( + splitType, + regexPattern, + pool, + new SecurityConfig(), + new BasicAuthenticationFactory().create(), + new LookupConfig() + ); + Assertions.assertDoesNotThrow(conversionFactory::conversion); + + splitType = "regex"; + conversionFactory = new ConversionFactory( + splitType, + regexPattern, + pool, + new SecurityConfig(), + new BasicAuthenticationFactory().create(), + new LookupConfig() + ); + Assertions.assertDoesNotThrow(conversionFactory::conversion); + + splitType = "none"; + conversionFactory = new ConversionFactory( + splitType, + regexPattern, + pool, + new SecurityConfig(), + new BasicAuthenticationFactory().create(), + new LookupConfig() + ); + Assertions.assertDoesNotThrow(conversionFactory::conversion); + } +} diff --git a/src/test/java/com/teragrep/lsh_01/CredentialsTest.java b/src/test/java/com/teragrep/lsh_01/CredentialsTest.java index e1e9572..48bc300 100644 --- a/src/test/java/com/teragrep/lsh_01/CredentialsTest.java +++ b/src/test/java/com/teragrep/lsh_01/CredentialsTest.java @@ -21,9 +21,9 @@ import com.codahale.metrics.MetricRegistry; import com.teragrep.lsh_01.authentication.BasicAuthentication; +import com.teragrep.lsh_01.conversion.RelpConversion; import com.teragrep.lsh_01.authentication.BasicAuthenticationFactory; import com.teragrep.lsh_01.config.LookupConfig; -import com.teragrep.lsh_01.config.PayloadConfig; import com.teragrep.lsh_01.config.RelpConfig; import com.teragrep.lsh_01.config.SecurityConfig; import com.teragrep.lsh_01.pool.*; @@ -62,8 +62,7 @@ public void testNoAuthRequired() { pool, securityConfig, basicAuthentication, - new LookupConfig(), - new PayloadConfig() + new LookupConfig() ); Assertions.assertFalse(relpConversion.requiresToken()); } @@ -90,8 +89,7 @@ public void testAuthRequired() { pool, securityConfig, basicAuthentication, - new LookupConfig(), - new PayloadConfig() + new LookupConfig() ); Assertions.assertTrue(relpConversion.requiresToken()); // FirstUser:VeryFirstPassword @@ -113,8 +111,7 @@ public void testValidBase64ButNoColon() { pool, securityConfig, basicAuthentication, - new LookupConfig(), - new PayloadConfig() + new LookupConfig() ); Assertions.assertTrue(relpConversion.requiresToken()); // Test @@ -136,8 +133,7 @@ public void testMultipleColons() { pool, securityConfig, basicAuthentication, - new LookupConfig(), - new PayloadConfig() + new LookupConfig() ); Assertions.assertTrue(relpConversion.requiresToken()); // UserWithColons:My:Password:Yay @@ -157,8 +153,7 @@ public void testInvalidBase64Auth() { pool, securityConfig, basicAuthentication, - new LookupConfig(), - new PayloadConfig() + new LookupConfig() ); Assertions.assertTrue(relpConversion.requiresToken()); IllegalArgumentException e = Assertions @@ -179,8 +174,7 @@ public void testNonBasicAuth() { pool, securityConfig, basicAuthentication, - new LookupConfig(), - new PayloadConfig() + new LookupConfig() ); Assertions.assertTrue(relpConversion.requiresToken()); IllegalArgumentException e = Assertions @@ -204,8 +198,7 @@ public void testWrongCredentials() { pool, securityConfig, basicAuthentication, - new LookupConfig(), - new PayloadConfig() + new LookupConfig() ); Assertions.assertTrue(relpConversion.requiresToken()); // SecondUser:WrongPassword -> Right user @@ -229,8 +222,7 @@ public void testEmptyUsername() { pool, securityConfig, basicAuthentication, - new LookupConfig(), - new PayloadConfig() + new LookupConfig() ); Assertions.assertTrue(relpConversion.requiresToken()); // :VeryFirstPassword -> Valid password, null username @@ -254,8 +246,7 @@ public void testEmptyPassword() { pool, securityConfig, basicAuthentication, - new LookupConfig(), - new PayloadConfig() + new LookupConfig() ); Assertions.assertTrue(relpConversion.requiresToken()); // FirstUser: -> Valid username, null password @@ -277,8 +268,7 @@ public void testNullToken() { pool, securityConfig, basicAuthentication, - new LookupConfig(), - new PayloadConfig() + new LookupConfig() ); Assertions.assertTrue(relpConversion.requiresToken()); IllegalArgumentException e = Assertions diff --git a/src/test/java/com/teragrep/lsh_01/EndToEndTest.java b/src/test/java/com/teragrep/lsh_01/EndToEndTest.java index 29f7f7f..85e841f 100644 --- a/src/test/java/com/teragrep/lsh_01/EndToEndTest.java +++ b/src/test/java/com/teragrep/lsh_01/EndToEndTest.java @@ -33,96 +33,44 @@ import java.util.List; import java.util.Random; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; @TestInstance(TestInstance.Lifecycle.PER_CLASS) public class EndToEndTest { private RelpServer relpServer; private NettyConfig nettyConfig; + private Thread program; @BeforeAll void setUp() throws InterruptedException { - System.setProperty("payload.splitEnabled", "true"); + System.setProperty("payload.splitType", "none"); System.setProperty("security.authRequired", "false"); System.setProperty("relp.port", "1601"); // Start listening to HTTP-requests - Thread program = new Thread(() -> Main.main(new String[] {})); + program = new Thread(() -> Main.main(new String[] {})); program.start(); Thread.sleep(3000); // wait for netty to start up - this.relpServer = new RelpServer(); - this.relpServer.setUpDefault(); + relpServer = new RelpServer(); + relpServer.setUpDefault(); - this.nettyConfig = new NettyConfig(); + nettyConfig = new NettyConfig(); } @AfterEach void reset() { - this.relpServer.clear(); + relpServer.clear(); } @AfterAll void tearDown() { - System.clearProperty("payload.splitEnabled"); + System.clearProperty("payload.splitType"); System.clearProperty("security.authRequired"); System.clearProperty("relp.port"); - this.relpServer.tearDown(); - } - - @Test - public void testSplittingMessage1() throws InterruptedException, ExecutionException { - String requestBody = "foofoo\nbar"; - - HttpClient httpClient = HttpClient.newHttpClient(); - - HttpRequest request = HttpRequest - .newBuilder(URI.create("http://" + nettyConfig.listenAddress + ":" + nettyConfig.listenPort)) - .POST(HttpRequest.BodyPublishers.ofString(requestBody)) - .build(); - - CompletableFuture> response = httpClient - .sendAsync(request, HttpResponse.BodyHandlers.ofString()); - - Assertions.assertEquals(200, response.get().statusCode()); - - List payloads = this.relpServer.payloads(); - - // assert that payload was correctly split into two - Assertions.assertEquals(2, payloads.size()); - Assertions.assertTrue(payloads.get(0).contains("foofoo")); - Assertions.assertFalse(payloads.get(0).contains("bar")); - Assertions.assertTrue(payloads.get(1).contains("bar")); - Assertions.assertFalse(payloads.get(1).contains("foofoo")); - } - - @Test - public void testSplittingMessage2() throws InterruptedException, ExecutionException { - String requestBody = "foofoo\nbar\nfoo bar"; - - HttpClient httpClient = HttpClient.newHttpClient(); - - HttpRequest request = HttpRequest - .newBuilder(URI.create("http://" + nettyConfig.listenAddress + ":" + nettyConfig.listenPort)) - .POST(HttpRequest.BodyPublishers.ofString(requestBody)) - .build(); - - CompletableFuture> response = httpClient - .sendAsync(request, HttpResponse.BodyHandlers.ofString()); - - Assertions.assertEquals(200, response.get().statusCode()); - - List payloads = this.relpServer.payloads(); - - // assert that payload was correctly split into three parts - Assertions.assertEquals(3, payloads.size()); - Assertions.assertTrue(payloads.get(0).contains("foofoo")); - Assertions.assertFalse(payloads.get(0).contains("bar")); - Assertions.assertTrue(payloads.get(1).contains("bar")); - Assertions.assertFalse(payloads.get(1).contains("foofoo")); - Assertions.assertTrue(payloads.get(2).contains("foo bar")); + relpServer.tearDown(); + program.interrupt(); } @Test @@ -152,7 +100,7 @@ public void testNullHeaders() { } @Test - public void testMultipleRequests() throws ExecutionException, InterruptedException { + public void testMultipleRequests() { ArrayList requestBodies = new ArrayList<>(); HttpClient httpClient = HttpClient.newHttpClient(); @@ -169,7 +117,8 @@ public void testMultipleRequests() throws ExecutionException, InterruptedExcepti CompletableFuture> response = httpClient .sendAsync(request, HttpResponse.BodyHandlers.ofString()); - Assertions.assertEquals(200, response.get().statusCode()); + int statusCode = Assertions.assertDoesNotThrow(() -> response.get().statusCode()); + Assertions.assertEquals(200, statusCode); } List payloads = this.relpServer.payloads(); diff --git a/src/test/java/com/teragrep/lsh_01/JsonPayloadTest.java b/src/test/java/com/teragrep/lsh_01/JsonPayloadTest.java new file mode 100644 index 0000000..3422975 --- /dev/null +++ b/src/test/java/com/teragrep/lsh_01/JsonPayloadTest.java @@ -0,0 +1,76 @@ +/* + logstash-http-input to syslog bridge + Copyright 2024 Suomen Kanuuna Oy + + Derivative Work of Elasticsearch + Copyright 2012-2015 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.teragrep.lsh_01; + +import com.teragrep.lsh_01.conversion.DefaultPayload; +import com.teragrep.lsh_01.conversion.JsonPayload; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; + +public class JsonPayloadTest { + + @Test + public void testSingleSplitting() { + String message = "{\"foo\": 1}"; + String requestBody = "[\n" + message + "\n]"; + + JsonPayload payload = new JsonPayload(new DefaultPayload(requestBody)); + List messages = payload.messages(); + + String expected = message.replaceAll("\\s", ""); + + Assertions.assertEquals(expected, messages.get(0)); + } + + @Test + public void testMultipleSplitting() { + String message1 = "{\"foo\": 1}"; + String message2 = "{\"bar\": 2}"; + String requestBody = "[\n" + message1 + ",\n" + message2 + "\n]"; + + JsonPayload payload = new JsonPayload(new DefaultPayload(requestBody)); + List messages = payload.messages(); + + String expected1 = message1.replaceAll("\\s", ""); + String expected2 = message2.replaceAll("\\s", ""); + + Assertions.assertEquals(2, messages.size()); + Assertions.assertEquals(expected1, messages.get(0)); + Assertions.assertEquals(expected2, messages.get(1)); + } + + @Test + public void testObjectEquality() { + String requestBody = "[\n{\"foo\": 1}\n]"; + String difRequestBody = "[\n{\"bar\": 2}\n]"; + JsonPayload payload = new JsonPayload(new DefaultPayload(requestBody)); + JsonPayload samePayload = new JsonPayload(new DefaultPayload(requestBody)); + JsonPayload difPayload = new JsonPayload(new DefaultPayload(difRequestBody)); + + // public methods of JsonPayload shouldn't affect an immutable object + payload.messages(); + + Assertions.assertEquals(payload, samePayload); + Assertions.assertNotEquals(payload, difPayload); + Assertions.assertNotEquals(samePayload, difPayload); + } +} diff --git a/src/test/java/com/teragrep/lsh_01/JsonSplittingTest.java b/src/test/java/com/teragrep/lsh_01/JsonSplittingTest.java new file mode 100644 index 0000000..8244dde --- /dev/null +++ b/src/test/java/com/teragrep/lsh_01/JsonSplittingTest.java @@ -0,0 +1,205 @@ +/* + logstash-http-input to syslog bridge + Copyright 2024 Suomen Kanuuna Oy + + Derivative Work of Elasticsearch + Copyright 2012-2015 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.teragrep.lsh_01; + +import com.teragrep.lsh_01.config.NettyConfig; +import com.teragrep.lsh_01.util.RelpServer; +import org.junit.jupiter.api.*; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class JsonSplittingTest { + + private RelpServer relpServer; + private Thread program; + + @BeforeAll + void setUp() throws InterruptedException { + // have to set config here before running Main + System.setProperty("payload.splitType", "json_array"); + System.setProperty("security.authRequired", "false"); + System.setProperty("relp.port", "1601"); + + // Start listening to HTTP-requests + program = new Thread(() -> Main.main(new String[] {})); + program.start(); + + Thread.sleep(3000); // wait for netty to start up + + this.relpServer = new RelpServer(); + this.relpServer.setUpDefault(); + } + + @AfterEach + void reset() { + this.relpServer.clear(); + } + + @AfterAll + void tearDown() { + System.clearProperty("payload.splitType"); + System.clearProperty("security.authRequired"); + System.clearProperty("relp.port"); + this.relpServer.tearDown(); + program.interrupt(); + } + + @Test + public void testJsonSplittingOneMessage() { // no splitting needed + String message = "{\"foo\": 1}"; + String requestBody = "[\n" + message + "\n]"; + + NettyConfig nettyConfig = new NettyConfig(); + + HttpClient httpClient = HttpClient.newHttpClient(); + + HttpRequest request = HttpRequest + .newBuilder(URI.create("http://" + nettyConfig.listenAddress + ":" + nettyConfig.listenPort)) + .POST(HttpRequest.BodyPublishers.ofString(requestBody)) + .build(); + + CompletableFuture> response = httpClient + .sendAsync(request, HttpResponse.BodyHandlers.ofString()); + + int statusCode = Assertions.assertDoesNotThrow(() -> response.get().statusCode()); + Assertions.assertEquals(200, statusCode); + + List payloads = this.relpServer.payloads(); + + String expected = message.replaceAll("\\s", ""); + + // assert that there is just one payload with the correct message + Assertions.assertEquals(1, payloads.size()); + Assertions.assertTrue(payloads.get(0).contains(expected)); + } + + @Test + public void testJsonSplittingTwoMessages() { + String message1 = "{\"foo\": 1}"; + String message2 = "{\"bar\": 2}"; + String requestBody = "[\n" + message1 + ",\n" + message2 + "\n]"; + + NettyConfig nettyConfig = new NettyConfig(); + + HttpClient httpClient = HttpClient.newHttpClient(); + + HttpRequest request = HttpRequest + .newBuilder(URI.create("http://" + nettyConfig.listenAddress + ":" + nettyConfig.listenPort)) + .POST(HttpRequest.BodyPublishers.ofString(requestBody)) + .build(); + + CompletableFuture> response = httpClient + .sendAsync(request, HttpResponse.BodyHandlers.ofString()); + + int statusCode = Assertions.assertDoesNotThrow(() -> response.get().statusCode()); + Assertions.assertEquals(200, statusCode); + + List payloads = this.relpServer.payloads(); + + String expected1 = message1.replaceAll("\\s", ""); + String expected2 = message2.replaceAll("\\s", ""); + + // assert that payload was correctly split + Assertions.assertEquals(2, payloads.size()); + Assertions.assertTrue(payloads.get(0).contains(expected1)); + Assertions.assertFalse(payloads.get(0).contains(expected2)); + + Assertions.assertTrue(payloads.get(1).contains(expected2)); + Assertions.assertFalse(payloads.get(1).contains(expected1)); + } + + @Test + public void testJsonSplittingThreeMessages() { + String message1 = "{\"foo\": 1}"; + String message2 = "{\"bar\": 2}"; + String message3 = "{\"foobar\": 3}"; + String requestBody = "[\n" + message1 + ",\n" + message2 + "\n, \n" + message3 + "\n]"; + + NettyConfig nettyConfig = new NettyConfig(); + + HttpClient httpClient = HttpClient.newHttpClient(); + + HttpRequest request = HttpRequest + .newBuilder(URI.create("http://" + nettyConfig.listenAddress + ":" + nettyConfig.listenPort)) + .POST(HttpRequest.BodyPublishers.ofString(requestBody)) + .build(); + + CompletableFuture> response = httpClient + .sendAsync(request, HttpResponse.BodyHandlers.ofString()); + + int statusCode = Assertions.assertDoesNotThrow(() -> response.get().statusCode()); + Assertions.assertEquals(200, statusCode); + + List payloads = this.relpServer.payloads(); + + String expected1 = message1.replaceAll("\\s", ""); + String expected2 = message2.replaceAll("\\s", ""); + String expected3 = message3.replaceAll("\\s", ""); + + // assert that payload was correctly split + Assertions.assertEquals(3, payloads.size()); + Assertions.assertTrue(payloads.get(0).contains(expected1)); + Assertions.assertFalse(payloads.get(0).contains(expected2)); + Assertions.assertFalse(payloads.get(0).contains(expected3)); + + Assertions.assertTrue(payloads.get(1).contains(expected2)); + Assertions.assertFalse(payloads.get(1).contains(expected1)); + Assertions.assertFalse(payloads.get(1).contains(expected3)); + + Assertions.assertTrue(payloads.get(2).contains(expected3)); + Assertions.assertFalse(payloads.get(2).contains(expected1)); + Assertions.assertFalse(payloads.get(2).contains(expected2)); + } + + @Test + public void testJsonSplittingNestedObjects() { + String payload = "{\"foo\": {\"bar\": 2}}"; + String requestBody = "[\n" + payload + "\n]"; + + NettyConfig nettyConfig = new NettyConfig(); + + HttpClient httpClient = HttpClient.newHttpClient(); + + HttpRequest request = HttpRequest + .newBuilder(URI.create("http://" + nettyConfig.listenAddress + ":" + nettyConfig.listenPort)) + .POST(HttpRequest.BodyPublishers.ofString(requestBody)) + .build(); + + CompletableFuture> response = httpClient + .sendAsync(request, HttpResponse.BodyHandlers.ofString()); + + int statusCode = Assertions.assertDoesNotThrow(() -> response.get().statusCode()); + Assertions.assertEquals(200, statusCode); + + List payloads = this.relpServer.payloads(); + + String expected = payload.replaceAll("\\s", ""); + + // assert that payload was correctly split + Assertions.assertEquals(1, payloads.size()); + Assertions.assertTrue(payloads.get(0).contains(expected)); + } +} diff --git a/src/test/java/com/teragrep/lsh_01/LookupTest.java b/src/test/java/com/teragrep/lsh_01/LookupTest.java index 8c06896..2639698 100644 --- a/src/test/java/com/teragrep/lsh_01/LookupTest.java +++ b/src/test/java/com/teragrep/lsh_01/LookupTest.java @@ -21,11 +21,11 @@ import com.codahale.metrics.MetricRegistry; import com.teragrep.jlt_01.StringLookupTable; +import com.teragrep.lsh_01.conversion.RelpConversion; import com.teragrep.lsh_01.authentication.BasicAuthentication; import com.teragrep.lsh_01.authentication.BasicAuthenticationFactory; import com.teragrep.lsh_01.authentication.Subject; import com.teragrep.lsh_01.config.LookupConfig; -import com.teragrep.lsh_01.config.PayloadConfig; import com.teragrep.lsh_01.config.RelpConfig; import com.teragrep.lsh_01.config.SecurityConfig; import com.teragrep.lsh_01.lookup.LookupTableFactory; @@ -69,8 +69,7 @@ public void testAppnameLookup() { pool, securityConfig, basicAuthentication, - new LookupConfig(), - new PayloadConfig() + new LookupConfig() ); // FirstUser:VeryFirstPassword! @@ -94,8 +93,7 @@ public void testHostnameLookup() { pool, securityConfig, basicAuthentication, - new LookupConfig(), - new PayloadConfig() + new LookupConfig() ); // FirstUser:VeryFirstPassword! @@ -121,8 +119,7 @@ public void testMissingLookups() { pool, securityConfig, basicAuthentication, - new LookupConfig(), - new PayloadConfig() + new LookupConfig() ); // MissingHostname:MyHostnameIsMissing diff --git a/src/test/java/com/teragrep/lsh_01/PayloadTest.java b/src/test/java/com/teragrep/lsh_01/PayloadTest.java index 369f76f..e69de29 100644 --- a/src/test/java/com/teragrep/lsh_01/PayloadTest.java +++ b/src/test/java/com/teragrep/lsh_01/PayloadTest.java @@ -1,108 +0,0 @@ -/* - logstash-http-input to syslog bridge - Copyright 2024 Suomen Kanuuna Oy - - Derivative Work of Elasticsearch - Copyright 2012-2015 Elasticsearch - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ -package com.teragrep.lsh_01; - -import com.teragrep.lsh_01.config.PayloadConfig; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.util.List; -import java.util.regex.Pattern; - -public class PayloadTest { - - @BeforeEach - public void addProperties() { - System.setProperty("payload.splitRegex", "\\n"); - System.setProperty("payload.splitEnabled", "false"); - } - - @AfterEach - public void cleanProperties() { - System.clearProperty("payload.splitRegex"); - System.clearProperty("payload.splitEnabled"); - } - - @Test - public void testDefaultSplitRegex() { - System.setProperty("payload.splitEnabled", "true"); - - String body = "foo\nbar\nfoobar"; - PayloadConfig payloadConfig = new PayloadConfig(); - Pattern splitPattern = Pattern.compile(payloadConfig.splitRegex); - Payload payload = new Payload(body, splitPattern); - List payloads = payload.split(); - - Assertions.assertEquals(3, payloads.size()); - Assertions.assertEquals("foo", payloads.get(0).take()); - Assertions.assertEquals("bar", payloads.get(1).take()); - Assertions.assertEquals("foobar", payloads.get(2).take()); - } - - @Test - public void testInvalidSplitRegex() { - System.setProperty("payload.splitEnabled", "true"); - System.setProperty("payload.splitRegex", "(a*b{)"); - - PayloadConfig payloadConfig = new PayloadConfig(); - Assertions.assertThrows(IllegalArgumentException.class, payloadConfig::validate); - } - - @Test - public void testValidSplitRegex() { - System.setProperty("payload.splitEnabled", "true"); - - PayloadConfig payloadConfig = new PayloadConfig(); - Assertions.assertDoesNotThrow(payloadConfig::validate); - } - - @Test - public void testCustomSplitRegex() { - System.setProperty("payload.splitRegex", ","); - System.setProperty("payload.splitEnabled", "true"); - - String body = "foo,bar,foobar"; - PayloadConfig payloadConfig = new PayloadConfig(); - Pattern splitPattern = Pattern.compile(payloadConfig.splitRegex); - Payload payload = new Payload(body, splitPattern); - List payloads = payload.split(); - - Assertions.assertEquals(3, payloads.size()); - Assertions.assertEquals("foo", payloads.get(0).take()); - Assertions.assertEquals("bar", payloads.get(1).take()); - Assertions.assertEquals("foobar", payloads.get(2).take()); - } - - @Test - public void testNoSplittingRequired() { - System.setProperty("payload.splitEnabled", "true"); - - String body = "foobar"; - PayloadConfig payloadConfig = new PayloadConfig(); - Pattern splitPattern = Pattern.compile(payloadConfig.splitRegex); - Payload payload = new Payload(body, splitPattern); - List payloads = payload.split(); - - Assertions.assertEquals(1, payloads.size()); - Assertions.assertEquals("foobar", payloads.get(0).take()); - } -} diff --git a/src/test/java/com/teragrep/lsh_01/RegexPayloadTest.java b/src/test/java/com/teragrep/lsh_01/RegexPayloadTest.java new file mode 100644 index 0000000..0f37860 --- /dev/null +++ b/src/test/java/com/teragrep/lsh_01/RegexPayloadTest.java @@ -0,0 +1,87 @@ +/* + logstash-http-input to syslog bridge + Copyright 2024 Suomen Kanuuna Oy + + Derivative Work of Elasticsearch + Copyright 2012-2015 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.teragrep.lsh_01; + +import com.teragrep.lsh_01.conversion.DefaultPayload; +import com.teragrep.lsh_01.conversion.RegexPayload; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.regex.Pattern; + +public class RegexPayloadTest { + + @Test + public void testDefaultSplitRegex() { + String body = "foo\nbar\nfoobar"; + Pattern splitPattern = Pattern.compile("\\n"); + RegexPayload payload = new RegexPayload(new DefaultPayload(body), splitPattern); + List messages = payload.messages(); + + Assertions.assertEquals(3, messages.size()); + Assertions.assertEquals("foo", messages.get(0)); + Assertions.assertEquals("bar", messages.get(1)); + Assertions.assertEquals("foobar", messages.get(2)); + } + + @Test + public void testCustomSplitRegex() { + String body = "foo,bar,foobar"; + Pattern splitPattern = Pattern.compile(","); + RegexPayload payload = new RegexPayload(new DefaultPayload(body), splitPattern); + List messages = payload.messages(); + + Assertions.assertEquals(3, messages.size()); + Assertions.assertEquals("foo", messages.get(0)); + Assertions.assertEquals("bar", messages.get(1)); + Assertions.assertEquals("foobar", messages.get(2)); + } + + @Test + public void testNoSplittingRequired() { + String body = "foobar"; + Pattern splitPattern = Pattern.compile("\\n"); + RegexPayload payload = new RegexPayload(new DefaultPayload(body), splitPattern); + List messages = payload.messages(); + + Assertions.assertEquals(1, messages.size()); + Assertions.assertEquals("foobar", messages.get(0)); + } + + @Test + public void testObjectEquality() { + Pattern splitPattern = Pattern.compile("\\n"); + Pattern splitPattern2 = Pattern.compile(","); + String requestBody = "[\n{\"foo\": 1}\n]"; + String difRequestBody = "[\n{\"bar\": 2}\n]"; + RegexPayload payload = new RegexPayload(new DefaultPayload(requestBody), splitPattern); + RegexPayload samePayload = new RegexPayload(new DefaultPayload(requestBody), splitPattern); + RegexPayload difPayload = new RegexPayload(new DefaultPayload(difRequestBody), splitPattern); + RegexPayload difPattern = new RegexPayload(new DefaultPayload(requestBody), splitPattern2); + + // public methods of JsonPayload shouldn't affect an immutable object + payload.messages(); + + Assertions.assertEquals(payload, samePayload); + Assertions.assertNotEquals(payload, difPayload); + Assertions.assertNotEquals(payload, difPattern); + } +} diff --git a/src/test/java/com/teragrep/lsh_01/RegexSplittingTest.java b/src/test/java/com/teragrep/lsh_01/RegexSplittingTest.java new file mode 100644 index 0000000..3ba36bd --- /dev/null +++ b/src/test/java/com/teragrep/lsh_01/RegexSplittingTest.java @@ -0,0 +1,126 @@ +/* + logstash-http-input to syslog bridge + Copyright 2024 Suomen Kanuuna Oy + + Derivative Work of Elasticsearch + Copyright 2012-2015 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.teragrep.lsh_01; + +import com.teragrep.lsh_01.config.NettyConfig; +import com.teragrep.lsh_01.util.RelpServer; +import org.junit.jupiter.api.*; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class RegexSplittingTest { + + private RelpServer relpServer; + private Thread program; + private NettyConfig nettyConfig; + + @BeforeAll + void setUp() throws InterruptedException { + System.setProperty("payload.splitType", "regex"); + System.setProperty("security.authRequired", "false"); + System.setProperty("relp.port", "1601"); + + // Start listening to HTTP-requests + program = new Thread(() -> Main.main(new String[] {})); + program.start(); + + Thread.sleep(3000); // wait for netty to start up + + this.relpServer = new RelpServer(); + this.relpServer.setUpDefault(); + + this.nettyConfig = new NettyConfig(); + } + + @AfterEach + void reset() { + this.relpServer.clear(); + } + + @AfterAll + void tearDown() { + System.clearProperty("payload.splitType"); + System.clearProperty("security.authRequired"); + System.clearProperty("relp.port"); + this.relpServer.tearDown(); + program.interrupt(); + } + + @Test + public void testRegexSplittingTwoMessages() { + String requestBody = "foofoo\nbar"; + + HttpClient httpClient = HttpClient.newHttpClient(); + + HttpRequest request = HttpRequest + .newBuilder(URI.create("http://" + nettyConfig.listenAddress + ":" + nettyConfig.listenPort)) + .POST(HttpRequest.BodyPublishers.ofString(requestBody)) + .build(); + + CompletableFuture> response = httpClient + .sendAsync(request, HttpResponse.BodyHandlers.ofString()); + + int statusCode = Assertions.assertDoesNotThrow(() -> response.get().statusCode()); + Assertions.assertEquals(200, statusCode); + + List payloads = this.relpServer.payloads(); + + // assert that payload was correctly split into two + Assertions.assertEquals(2, payloads.size()); + Assertions.assertTrue(payloads.get(0).contains("foofoo")); + Assertions.assertFalse(payloads.get(0).contains("bar")); + Assertions.assertTrue(payloads.get(1).contains("bar")); + Assertions.assertFalse(payloads.get(1).contains("foofoo")); + } + + @Test + public void testRegexSplittingThreeMessages() { + String requestBody = "foofoo\nbar\nfoo bar"; + + HttpClient httpClient = HttpClient.newHttpClient(); + + HttpRequest request = HttpRequest + .newBuilder(URI.create("http://" + nettyConfig.listenAddress + ":" + nettyConfig.listenPort)) + .POST(HttpRequest.BodyPublishers.ofString(requestBody)) + .build(); + + CompletableFuture> response = httpClient + .sendAsync(request, HttpResponse.BodyHandlers.ofString()); + + int statusCode = Assertions.assertDoesNotThrow(() -> response.get().statusCode()); + Assertions.assertEquals(200, statusCode); + + List payloads = this.relpServer.payloads(); + + // assert that payload was correctly split into three parts + Assertions.assertEquals(3, payloads.size()); + Assertions.assertTrue(payloads.get(0).contains("foofoo")); + Assertions.assertFalse(payloads.get(0).contains("bar")); + Assertions.assertTrue(payloads.get(1).contains("bar")); + Assertions.assertFalse(payloads.get(1).contains("foofoo")); + Assertions.assertTrue(payloads.get(2).contains("foo bar")); + } +} From 48f022e6e1eb8751fa10a598605732ed0b1c17e2 Mon Sep 17 00:00:00 2001 From: Rasmus Jokinen <146736881+51-code@users.noreply.github.com> Date: Wed, 9 Oct 2024 09:11:15 +0300 Subject: [PATCH 02/10] Fix rebase --- src/main/java/com/teragrep/lsh_01/Main.java | 12 ++++-------- .../{ => conversion}/MetricRelpConversion.java | 2 +- .../teragrep/lsh_01/pool/RelpConnectionFactory.java | 4 ++++ src/test/java/com/teragrep/lsh_01/MetricTest.java | 6 ++++-- src/test/java/com/teragrep/lsh_01/PayloadTest.java | 0 5 files changed, 13 insertions(+), 11 deletions(-) rename src/main/java/com/teragrep/lsh_01/{ => conversion}/MetricRelpConversion.java (98%) delete mode 100644 src/test/java/com/teragrep/lsh_01/PayloadTest.java diff --git a/src/main/java/com/teragrep/lsh_01/Main.java b/src/main/java/com/teragrep/lsh_01/Main.java index eb3126d..bcb99a9 100644 --- a/src/main/java/com/teragrep/lsh_01/Main.java +++ b/src/main/java/com/teragrep/lsh_01/Main.java @@ -84,14 +84,10 @@ public static void main(String[] args) { RelpConnectionFactory relpConnectionFactory = new RelpConnectionFactory(relpConfig, metricRegistry); Pool pool = new Pool<>(relpConnectionFactory, new ManagedRelpConnectionStub()); - IMessageHandler conversion = new MetricRelpConversion(new ConversionFactory( - propsMap, - pool, - securityConfig, - basicAuthentication, - lookupConfig - ).conversion(), - metricRegistry); + IMessageHandler conversion = new MetricRelpConversion( + new ConversionFactory(propsMap, pool, securityConfig, basicAuthentication, lookupConfig).conversion(), + metricRegistry + ); try ( HttpServer server = new MetricHttpServer( diff --git a/src/main/java/com/teragrep/lsh_01/MetricRelpConversion.java b/src/main/java/com/teragrep/lsh_01/conversion/MetricRelpConversion.java similarity index 98% rename from src/main/java/com/teragrep/lsh_01/MetricRelpConversion.java rename to src/main/java/com/teragrep/lsh_01/conversion/MetricRelpConversion.java index 9d58257..7843577 100644 --- a/src/main/java/com/teragrep/lsh_01/MetricRelpConversion.java +++ b/src/main/java/com/teragrep/lsh_01/conversion/MetricRelpConversion.java @@ -17,7 +17,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package com.teragrep.lsh_01; +package com.teragrep.lsh_01.conversion; import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.SlidingWindowReservoir; diff --git a/src/main/java/com/teragrep/lsh_01/pool/RelpConnectionFactory.java b/src/main/java/com/teragrep/lsh_01/pool/RelpConnectionFactory.java index 86f7940..83ebb14 100644 --- a/src/main/java/com/teragrep/lsh_01/pool/RelpConnectionFactory.java +++ b/src/main/java/com/teragrep/lsh_01/pool/RelpConnectionFactory.java @@ -30,6 +30,10 @@ public class RelpConnectionFactory implements Supplier { private final RelpConfig relpConfig; private final MetricRegistry metricRegistry; + public RelpConnectionFactory(RelpConfig relpConfig) { // for testing, new metric registry can be used + this(relpConfig, new MetricRegistry()); + } + public RelpConnectionFactory(RelpConfig relpConfig, MetricRegistry metricRegistry) { this.relpConfig = relpConfig; this.metricRegistry = metricRegistry; diff --git a/src/test/java/com/teragrep/lsh_01/MetricTest.java b/src/test/java/com/teragrep/lsh_01/MetricTest.java index 11a829f..4959e2b 100644 --- a/src/test/java/com/teragrep/lsh_01/MetricTest.java +++ b/src/test/java/com/teragrep/lsh_01/MetricTest.java @@ -25,9 +25,11 @@ import com.teragrep.lsh_01.authentication.BasicAuthenticationFactory; import com.teragrep.lsh_01.authentication.SubjectAnonymous; import com.teragrep.lsh_01.config.LookupConfig; -import com.teragrep.lsh_01.config.PayloadConfig; import com.teragrep.lsh_01.config.RelpConfig; import com.teragrep.lsh_01.config.SecurityConfig; +import com.teragrep.lsh_01.conversion.IMessageHandler; +import com.teragrep.lsh_01.conversion.MetricRelpConversion; +import com.teragrep.lsh_01.conversion.RelpConversion; import com.teragrep.lsh_01.fakes.RelpConnectionFactoryFake; import com.teragrep.lsh_01.fakes.RelpConnectionFake; import com.teragrep.lsh_01.fakes.ResendingRelpConnectionFake; @@ -205,7 +207,7 @@ public void testSendLatencyMetric() { // latency of the whole process, message i // the message processing starts from RelpConversion IMessageHandler relpConversion = new MetricRelpConversion( - new RelpConversion(new Pool<>(relpConnectionFactory, new ManagedRelpConnectionStub()), new SecurityConfig(), new BasicAuthenticationFactory().create(), new LookupConfig(), new PayloadConfig()), registry + new RelpConversion(new Pool<>(relpConnectionFactory, new ManagedRelpConnectionStub()), new SecurityConfig(), new BasicAuthenticationFactory().create(), new LookupConfig()), registry ); for (int i = 0; i < messages; i++) { diff --git a/src/test/java/com/teragrep/lsh_01/PayloadTest.java b/src/test/java/com/teragrep/lsh_01/PayloadTest.java deleted file mode 100644 index e69de29..0000000 From 19729d2b31fe9e5e645bd9861a69ef86a8a52761 Mon Sep 17 00:00:00 2001 From: Rasmus Jokinen <146736881+51-code@users.noreply.github.com> Date: Wed, 9 Oct 2024 09:32:32 +0300 Subject: [PATCH 03/10] Use try-with-resources when loading configuration file --- .../java/com/teragrep/lsh_01/config/PathProperties.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/teragrep/lsh_01/config/PathProperties.java b/src/main/java/com/teragrep/lsh_01/config/PathProperties.java index 727f83b..17546e8 100644 --- a/src/main/java/com/teragrep/lsh_01/config/PathProperties.java +++ b/src/main/java/com/teragrep/lsh_01/config/PathProperties.java @@ -20,8 +20,9 @@ package com.teragrep.lsh_01.config; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; import java.util.Collections; import java.util.Map; import java.util.Properties; @@ -42,7 +43,9 @@ public PathProperties(File file) { @Override public Map deepCopyAsUnmodifiableMap() throws IOException { Properties properties = new Properties(); - properties.load(new FileInputStream(file)); + try (final InputStream in = Files.newInputStream(file.toPath())) { + properties.load(in); + } return Collections .unmodifiableMap( properties From d750c72f27f0c6118772dd96ddd88e5cd0c41bfd Mon Sep 17 00:00:00 2001 From: Rasmus Jokinen <146736881+51-code@users.noreply.github.com> Date: Wed, 9 Oct 2024 10:40:02 +0300 Subject: [PATCH 04/10] Add object equality tests for ConversionFactory --- .../authentication/BasicAuthentication.java | 11 ++++ .../lsh_01/config/ConversionFactory.java | 12 ++++ .../teragrep/lsh_01/config/LookupConfig.java | 10 +++ .../lsh_01/config/SecurityConfig.java | 10 +++ .../java/com/teragrep/lsh_01/pool/Pool.java | 10 +++ .../lsh_01/ConversionFactoryTest.java | 63 +++++++++++++++++++ 6 files changed, 116 insertions(+) diff --git a/src/main/java/com/teragrep/lsh_01/authentication/BasicAuthentication.java b/src/main/java/com/teragrep/lsh_01/authentication/BasicAuthentication.java index 1daafac..01956a1 100644 --- a/src/main/java/com/teragrep/lsh_01/authentication/BasicAuthentication.java +++ b/src/main/java/com/teragrep/lsh_01/authentication/BasicAuthentication.java @@ -67,4 +67,15 @@ public Subject asSubject(String token) { return subjectStub; } } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + final BasicAuthentication cast = (BasicAuthentication) o; + // equality of CredentialLookup can't be checked. It has no equals() function and the Map is only traversable by key. + return decoder.equals(cast.decoder); + } } diff --git a/src/main/java/com/teragrep/lsh_01/config/ConversionFactory.java b/src/main/java/com/teragrep/lsh_01/config/ConversionFactory.java index 0a6bde9..2a50d7f 100644 --- a/src/main/java/com/teragrep/lsh_01/config/ConversionFactory.java +++ b/src/main/java/com/teragrep/lsh_01/config/ConversionFactory.java @@ -127,4 +127,16 @@ private void validateConfiguration() { ); } } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + final ConversionFactory cast = (ConversionFactory) o; + return splitType.equals(cast.splitType) && regexPattern.equals(cast.regexPattern) && pool + .equals(cast.pool) && securityConfig.equals(cast.securityConfig) + && basicAuthentication.equals(cast.basicAuthentication) && lookupConfig.equals(cast.lookupConfig); + } } diff --git a/src/main/java/com/teragrep/lsh_01/config/LookupConfig.java b/src/main/java/com/teragrep/lsh_01/config/LookupConfig.java index b3059fa..ca3251c 100644 --- a/src/main/java/com/teragrep/lsh_01/config/LookupConfig.java +++ b/src/main/java/com/teragrep/lsh_01/config/LookupConfig.java @@ -40,4 +40,14 @@ public void validate() { public String toString() { return "LookupConfig{" + "hostnamePath='" + hostnamePath + '\'' + ", appNamePath='" + appNamePath + '\'' + '}'; } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + final LookupConfig cast = (LookupConfig) o; + return hostnamePath.equals(cast.hostnamePath) && appNamePath.equals(cast.appNamePath); + } } diff --git a/src/main/java/com/teragrep/lsh_01/config/SecurityConfig.java b/src/main/java/com/teragrep/lsh_01/config/SecurityConfig.java index cfca2cb..372e2b1 100644 --- a/src/main/java/com/teragrep/lsh_01/config/SecurityConfig.java +++ b/src/main/java/com/teragrep/lsh_01/config/SecurityConfig.java @@ -34,4 +34,14 @@ public SecurityConfig() { public void validate() { } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + final SecurityConfig cast = (SecurityConfig) o; + return authRequired == cast.authRequired; + } } diff --git a/src/main/java/com/teragrep/lsh_01/pool/Pool.java b/src/main/java/com/teragrep/lsh_01/pool/Pool.java index e3808e4..72a2eb6 100644 --- a/src/main/java/com/teragrep/lsh_01/pool/Pool.java +++ b/src/main/java/com/teragrep/lsh_01/pool/Pool.java @@ -109,4 +109,14 @@ public void close() { offer(stub); } + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + final Pool cast = (Pool) o; + return supplier.equals(cast.supplier) && queue.equals(cast.queue) && stub.equals(cast.stub) + && lock.equals(cast.lock) && (close.get() == cast.close.get()); + } } diff --git a/src/test/java/com/teragrep/lsh_01/ConversionFactoryTest.java b/src/test/java/com/teragrep/lsh_01/ConversionFactoryTest.java index eade9b3..661e45c 100644 --- a/src/test/java/com/teragrep/lsh_01/ConversionFactoryTest.java +++ b/src/test/java/com/teragrep/lsh_01/ConversionFactoryTest.java @@ -143,4 +143,67 @@ public void testValidSplitType() { ); Assertions.assertDoesNotThrow(conversionFactory::conversion); } + + @Test + public void testEqualConversionFactories() { + String regexPattern = ""; + String splitType = "json_array"; + + RelpConnectionFactory relpConnectionFactory = new RelpConnectionFactory(new RelpConfig()); + Pool pool = new Pool<>(relpConnectionFactory, new ManagedRelpConnectionStub()); + + ConversionFactory conversionFactory = new ConversionFactory( + splitType, + regexPattern, + pool, + new SecurityConfig(), + new BasicAuthenticationFactory().create(), + new LookupConfig() + ); + + ConversionFactory conversionFactoryCopy = new ConversionFactory( + splitType, + regexPattern, + pool, + new SecurityConfig(), + new BasicAuthenticationFactory().create(), + new LookupConfig() + ); + + // calling functions should have no effect on an immutable object + conversionFactory.conversion(); + + Assertions.assertEquals(conversionFactory, conversionFactoryCopy); + } + + @Test + public void testNotEqualConversionFactories() { + String regexPattern = ""; + String splitType = "json_array"; + + RelpConnectionFactory relpConnectionFactory = new RelpConnectionFactory(new RelpConfig()); + Pool pool = new Pool<>(relpConnectionFactory, new ManagedRelpConnectionStub()); + + ConversionFactory conversionFactory = new ConversionFactory( + splitType, + regexPattern, + pool, + new SecurityConfig(), + new BasicAuthenticationFactory().create(), + new LookupConfig() + ); + + regexPattern = "\n"; + splitType = "regex"; + ConversionFactory conversionFactoryCopy = new ConversionFactory( + splitType, + regexPattern, + pool, + new SecurityConfig(), + new BasicAuthenticationFactory().create(), + new LookupConfig() + ); + + Assertions.assertNotEquals(conversionFactory, conversionFactoryCopy); + } } From daa42eaf0bf644596079d052bd6d2603632051f1 Mon Sep 17 00:00:00 2001 From: Rasmus Jokinen <146736881+51-code@users.noreply.github.com> Date: Wed, 9 Oct 2024 12:35:07 +0300 Subject: [PATCH 05/10] Split object equality test into two tests in RegexPayloadTest --- .../com/teragrep/lsh_01/RegexPayloadTest.java | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/test/java/com/teragrep/lsh_01/RegexPayloadTest.java b/src/test/java/com/teragrep/lsh_01/RegexPayloadTest.java index 0f37860..6dc734b 100644 --- a/src/test/java/com/teragrep/lsh_01/RegexPayloadTest.java +++ b/src/test/java/com/teragrep/lsh_01/RegexPayloadTest.java @@ -67,20 +67,28 @@ public void testNoSplittingRequired() { } @Test - public void testObjectEquality() { + public void testObjectEquals() { Pattern splitPattern = Pattern.compile("\\n"); - Pattern splitPattern2 = Pattern.compile(","); String requestBody = "[\n{\"foo\": 1}\n]"; - String difRequestBody = "[\n{\"bar\": 2}\n]"; RegexPayload payload = new RegexPayload(new DefaultPayload(requestBody), splitPattern); RegexPayload samePayload = new RegexPayload(new DefaultPayload(requestBody), splitPattern); - RegexPayload difPayload = new RegexPayload(new DefaultPayload(difRequestBody), splitPattern); - RegexPayload difPattern = new RegexPayload(new DefaultPayload(requestBody), splitPattern2); // public methods of JsonPayload shouldn't affect an immutable object payload.messages(); Assertions.assertEquals(payload, samePayload); + } + + @Test + public void testObjectNotEquals() { + Pattern splitPattern = Pattern.compile("\\n"); + Pattern difSplitPattern = Pattern.compile(","); + String requestBody = "[\n{\"foo\": 1}\n]"; + String difRequestBody = "[\n{\"bar\": 2}\n]"; + RegexPayload payload = new RegexPayload(new DefaultPayload(requestBody), splitPattern); + RegexPayload difPayload = new RegexPayload(new DefaultPayload(difRequestBody), splitPattern); + RegexPayload difPattern = new RegexPayload(new DefaultPayload(requestBody), difSplitPattern); + Assertions.assertNotEquals(payload, difPayload); Assertions.assertNotEquals(payload, difPattern); } From 2472f7d0fe282de87b5bda3a4d0d7061e8c4e628 Mon Sep 17 00:00:00 2001 From: Rasmus Jokinen <146736881+51-code@users.noreply.github.com> Date: Thu, 10 Oct 2024 13:50:51 +0300 Subject: [PATCH 06/10] Added missing equals and hashCode functions, added equality tests --- pom.xml | 1 + .../authentication/BasicAuthentication.java | 9 ++- .../lsh_01/config/PathProperties.java | 16 ++++ .../lsh_01/config/SecurityConfig.java | 7 ++ .../lsh_01/conversion/DefaultPayload.java | 6 ++ .../lsh_01/conversion/JsonPayload.java | 6 ++ .../lsh_01/conversion/RegexPayload.java | 6 ++ .../java/com/teragrep/lsh_01/pool/Pool.java | 11 --- .../lsh_01/BasicAuthenticationTest.java | 77 +++++++++++++++++++ .../lsh_01/ConversionFactoryTest.java | 6 +- .../teragrep/lsh_01/DefaultPayloadTest.java | 56 ++++++++++++++ .../com/teragrep/lsh_01/JsonPayloadTest.java | 26 ++++++- .../teragrep/lsh_01/PathPropertiesTest.java | 57 ++++++++++++++ .../com/teragrep/lsh_01/RegexPayloadTest.java | 16 ++++ .../teragrep/lsh_01/SecurityConfigTest.java | 64 +++++++++++++++ src/test/resources/exampleCredentials.json | 6 ++ .../properties/customTest.properties | 26 +++++++ .../properties/defaultTest.properties | 26 +++++++ 18 files changed, 403 insertions(+), 19 deletions(-) create mode 100644 src/test/java/com/teragrep/lsh_01/BasicAuthenticationTest.java create mode 100644 src/test/java/com/teragrep/lsh_01/DefaultPayloadTest.java create mode 100644 src/test/java/com/teragrep/lsh_01/PathPropertiesTest.java create mode 100644 src/test/java/com/teragrep/lsh_01/SecurityConfigTest.java create mode 100644 src/test/resources/exampleCredentials.json create mode 100644 src/test/resources/properties/customTest.properties create mode 100644 src/test/resources/properties/defaultTest.properties diff --git a/pom.xml b/pom.xml index a95d535..14f5dc9 100644 --- a/pom.xml +++ b/pom.xml @@ -242,6 +242,7 @@ src/main/assembly/jar-with-dependencies.xml src/main/resources/* src/test/resources/certificates/* + src/test/resources/properties/* README.adoc diff --git a/src/main/java/com/teragrep/lsh_01/authentication/BasicAuthentication.java b/src/main/java/com/teragrep/lsh_01/authentication/BasicAuthentication.java index 01956a1..49d36ba 100644 --- a/src/main/java/com/teragrep/lsh_01/authentication/BasicAuthentication.java +++ b/src/main/java/com/teragrep/lsh_01/authentication/BasicAuthentication.java @@ -24,6 +24,7 @@ import org.apache.logging.log4j.Logger; import java.util.Base64; +import java.util.Objects; public class BasicAuthentication { @@ -75,7 +76,11 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; final BasicAuthentication cast = (BasicAuthentication) o; - // equality of CredentialLookup can't be checked. It has no equals() function and the Map is only traversable by key. - return decoder.equals(cast.decoder); + return decoder.equals(cast.decoder) && credentialLookup.equals(cast.credentialLookup); + } + + @Override + public int hashCode() { + return Objects.hash(decoder, credentialLookup); } } diff --git a/src/main/java/com/teragrep/lsh_01/config/PathProperties.java b/src/main/java/com/teragrep/lsh_01/config/PathProperties.java index 17546e8..e91a6eb 100644 --- a/src/main/java/com/teragrep/lsh_01/config/PathProperties.java +++ b/src/main/java/com/teragrep/lsh_01/config/PathProperties.java @@ -25,6 +25,7 @@ import java.nio.file.Files; import java.util.Collections; import java.util.Map; +import java.util.Objects; import java.util.Properties; import java.util.stream.Collectors; @@ -54,4 +55,19 @@ public Map deepCopyAsUnmodifiableMap() throws IOException { .collect(Collectors.toMap(k -> k.getKey().toString(), k -> k.getValue().toString())) ); } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + PathProperties that = (PathProperties) o; + return file.equals(that.file); + } + + @Override + public int hashCode() { + return Objects.hashCode(file); + } } diff --git a/src/main/java/com/teragrep/lsh_01/config/SecurityConfig.java b/src/main/java/com/teragrep/lsh_01/config/SecurityConfig.java index 372e2b1..fd020f3 100644 --- a/src/main/java/com/teragrep/lsh_01/config/SecurityConfig.java +++ b/src/main/java/com/teragrep/lsh_01/config/SecurityConfig.java @@ -19,6 +19,8 @@ */ package com.teragrep.lsh_01.config; +import java.util.Objects; + public class SecurityConfig implements Validateable { public final boolean authRequired; @@ -44,4 +46,9 @@ public boolean equals(Object o) { final SecurityConfig cast = (SecurityConfig) o; return authRequired == cast.authRequired; } + + @Override + public int hashCode() { + return Objects.hashCode(authRequired); + } } diff --git a/src/main/java/com/teragrep/lsh_01/conversion/DefaultPayload.java b/src/main/java/com/teragrep/lsh_01/conversion/DefaultPayload.java index 3065267..b95e944 100644 --- a/src/main/java/com/teragrep/lsh_01/conversion/DefaultPayload.java +++ b/src/main/java/com/teragrep/lsh_01/conversion/DefaultPayload.java @@ -21,6 +21,7 @@ import java.util.Collections; import java.util.List; +import java.util.Objects; public final class DefaultPayload implements Payload { @@ -46,4 +47,9 @@ public boolean equals(final Object object) { final DefaultPayload cast = (DefaultPayload) object; return message.equals(cast.message); } + + @Override + public int hashCode() { + return Objects.hashCode(message); + } } diff --git a/src/main/java/com/teragrep/lsh_01/conversion/JsonPayload.java b/src/main/java/com/teragrep/lsh_01/conversion/JsonPayload.java index 18b0c56..64be5bd 100644 --- a/src/main/java/com/teragrep/lsh_01/conversion/JsonPayload.java +++ b/src/main/java/com/teragrep/lsh_01/conversion/JsonPayload.java @@ -24,6 +24,7 @@ import java.io.StringReader; import java.util.ArrayList; import java.util.List; +import java.util.Objects; /** * A Json array payload splittable into individual json objects. @@ -67,4 +68,9 @@ public boolean equals(final Object object) { final JsonPayload cast = (JsonPayload) object; return payload.equals(cast.payload); } + + @Override + public int hashCode() { + return Objects.hashCode(payload); + } } diff --git a/src/main/java/com/teragrep/lsh_01/conversion/RegexPayload.java b/src/main/java/com/teragrep/lsh_01/conversion/RegexPayload.java index 64abb15..84a3a7b 100644 --- a/src/main/java/com/teragrep/lsh_01/conversion/RegexPayload.java +++ b/src/main/java/com/teragrep/lsh_01/conversion/RegexPayload.java @@ -21,6 +21,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.regex.Pattern; /** @@ -64,4 +65,9 @@ public boolean equals(final Object object) { final RegexPayload cast = (RegexPayload) object; return payload.equals(cast.payload) && splitPattern.equals(cast.splitPattern); } + + @Override + public int hashCode() { + return Objects.hash(payload, splitPattern); + } } diff --git a/src/main/java/com/teragrep/lsh_01/pool/Pool.java b/src/main/java/com/teragrep/lsh_01/pool/Pool.java index 72a2eb6..8bbb8a2 100644 --- a/src/main/java/com/teragrep/lsh_01/pool/Pool.java +++ b/src/main/java/com/teragrep/lsh_01/pool/Pool.java @@ -108,15 +108,4 @@ public void close() { // close all that are in the pool right now offer(stub); } - - @Override - public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - final Pool cast = (Pool) o; - return supplier.equals(cast.supplier) && queue.equals(cast.queue) && stub.equals(cast.stub) - && lock.equals(cast.lock) && (close.get() == cast.close.get()); - } } diff --git a/src/test/java/com/teragrep/lsh_01/BasicAuthenticationTest.java b/src/test/java/com/teragrep/lsh_01/BasicAuthenticationTest.java new file mode 100644 index 0000000..e361f53 --- /dev/null +++ b/src/test/java/com/teragrep/lsh_01/BasicAuthenticationTest.java @@ -0,0 +1,77 @@ +/* + logstash-http-input to syslog bridge + Copyright 2024 Suomen Kanuuna Oy + + Derivative Work of Elasticsearch + Copyright 2012-2015 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.teragrep.lsh_01; + +import com.teragrep.jai_02.CredentialLookup; +import com.teragrep.lsh_01.authentication.BasicAuthentication; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.BufferedReader; +import java.io.FileReader; + +public class BasicAuthenticationTest { + + @Test + public void testEquals() { + String credentialsFile = "src/test/resources/credentials.json"; + BufferedReader br = Assertions.assertDoesNotThrow(() -> new BufferedReader(new FileReader(credentialsFile))); + CredentialLookup credentials = new CredentialLookup(br); + + BasicAuthentication auth1 = new BasicAuthentication(credentials); + BasicAuthentication auth2 = new BasicAuthentication(credentials); + + Assertions.assertEquals(auth1, auth2); + } + + @Test + public void testNotEquals() { + String credentialsFile = "src/test/resources/credentials.json"; + BufferedReader br = Assertions.assertDoesNotThrow(() -> new BufferedReader(new FileReader(credentialsFile))); + CredentialLookup credentials = new CredentialLookup(br); + + String credentialsFile2 = "src/test/resources/exampleCredentials.json"; + BufferedReader br2 = Assertions.assertDoesNotThrow(() -> new BufferedReader(new FileReader(credentialsFile2))); + CredentialLookup credentials2 = new CredentialLookup(br2); + + BasicAuthentication auth1 = new BasicAuthentication(credentials); + BasicAuthentication auth2 = new BasicAuthentication(credentials2); + + Assertions.assertNotEquals(auth1, auth2); + } + + @Test + public void testHashCode() { + String credentialsFile = "src/test/resources/credentials.json"; + BufferedReader br = Assertions.assertDoesNotThrow(() -> new BufferedReader(new FileReader(credentialsFile))); + CredentialLookup credentials = new CredentialLookup(br); + + String credentialsFile2 = "src/test/resources/exampleCredentials.json"; + BufferedReader br2 = Assertions.assertDoesNotThrow(() -> new BufferedReader(new FileReader(credentialsFile2))); + CredentialLookup credentials2 = new CredentialLookup(br2); + + BasicAuthentication auth1 = new BasicAuthentication(credentials); + BasicAuthentication auth2 = new BasicAuthentication(credentials); + BasicAuthentication auth3 = new BasicAuthentication(credentials2); + + Assertions.assertEquals(auth1.hashCode(), auth2.hashCode()); + Assertions.assertNotEquals(auth1.hashCode(), auth3.hashCode()); + } +} diff --git a/src/test/java/com/teragrep/lsh_01/ConversionFactoryTest.java b/src/test/java/com/teragrep/lsh_01/ConversionFactoryTest.java index 661e45c..f884c9f 100644 --- a/src/test/java/com/teragrep/lsh_01/ConversionFactoryTest.java +++ b/src/test/java/com/teragrep/lsh_01/ConversionFactoryTest.java @@ -19,6 +19,7 @@ */ package com.teragrep.lsh_01; +import com.teragrep.lsh_01.authentication.BasicAuthentication; import com.teragrep.lsh_01.authentication.BasicAuthenticationFactory; import com.teragrep.lsh_01.config.*; import com.teragrep.lsh_01.pool.IManagedRelpConnection; @@ -151,13 +152,14 @@ public void testEqualConversionFactories() { RelpConnectionFactory relpConnectionFactory = new RelpConnectionFactory(new RelpConfig()); Pool pool = new Pool<>(relpConnectionFactory, new ManagedRelpConnectionStub()); + BasicAuthentication auth = new BasicAuthenticationFactory().create(); ConversionFactory conversionFactory = new ConversionFactory( splitType, regexPattern, pool, new SecurityConfig(), - new BasicAuthenticationFactory().create(), + auth, new LookupConfig() ); @@ -166,7 +168,7 @@ public void testEqualConversionFactories() { regexPattern, pool, new SecurityConfig(), - new BasicAuthenticationFactory().create(), + auth, new LookupConfig() ); diff --git a/src/test/java/com/teragrep/lsh_01/DefaultPayloadTest.java b/src/test/java/com/teragrep/lsh_01/DefaultPayloadTest.java new file mode 100644 index 0000000..e8c1875 --- /dev/null +++ b/src/test/java/com/teragrep/lsh_01/DefaultPayloadTest.java @@ -0,0 +1,56 @@ +/* + logstash-http-input to syslog bridge + Copyright 2024 Suomen Kanuuna Oy + + Derivative Work of Elasticsearch + Copyright 2012-2015 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.teragrep.lsh_01; + +import com.teragrep.lsh_01.conversion.DefaultPayload; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class DefaultPayloadTest { + + @Test + public void testEquals() { + DefaultPayload payload1 = new DefaultPayload("payload"); + DefaultPayload payload2 = new DefaultPayload("payload"); + + // calling functions shouldn't have effect on an immutable object + payload1.messages(); + + Assertions.assertEquals(payload1, payload2); + } + + @Test + public void testNotEquals() { + DefaultPayload payload1 = new DefaultPayload("payload"); + DefaultPayload payload2 = new DefaultPayload(""); + + Assertions.assertNotEquals(payload1, payload2); + } + + @Test + public void testHashCode() { + DefaultPayload payload1 = new DefaultPayload("payload"); + DefaultPayload payload2 = new DefaultPayload("payload"); + DefaultPayload payload3 = new DefaultPayload(""); + + Assertions.assertEquals(payload1.hashCode(), payload2.hashCode()); + Assertions.assertNotEquals(payload1.hashCode(), payload3.hashCode()); + } +} diff --git a/src/test/java/com/teragrep/lsh_01/JsonPayloadTest.java b/src/test/java/com/teragrep/lsh_01/JsonPayloadTest.java index 3422975..17a8d47 100644 --- a/src/test/java/com/teragrep/lsh_01/JsonPayloadTest.java +++ b/src/test/java/com/teragrep/lsh_01/JsonPayloadTest.java @@ -59,18 +59,36 @@ public void testMultipleSplitting() { } @Test - public void testObjectEquality() { + public void testEquals() { String requestBody = "[\n{\"foo\": 1}\n]"; - String difRequestBody = "[\n{\"bar\": 2}\n]"; JsonPayload payload = new JsonPayload(new DefaultPayload(requestBody)); JsonPayload samePayload = new JsonPayload(new DefaultPayload(requestBody)); - JsonPayload difPayload = new JsonPayload(new DefaultPayload(difRequestBody)); // public methods of JsonPayload shouldn't affect an immutable object payload.messages(); Assertions.assertEquals(payload, samePayload); + } + + @Test + public void testNotEquals() { + String requestBody = "[\n{\"foo\": 1}\n]"; + String difRequestBody = "[\n{\"bar\": 2}\n]"; + JsonPayload payload = new JsonPayload(new DefaultPayload(requestBody)); + JsonPayload difPayload = new JsonPayload(new DefaultPayload(difRequestBody)); + Assertions.assertNotEquals(payload, difPayload); - Assertions.assertNotEquals(samePayload, difPayload); + } + + @Test + public void testHashCode() { + String requestBody = "[\n{\"foo\": 1}\n]"; + String difRequestBody = "[\n{\"bar\": 2}\n]"; + JsonPayload payload = new JsonPayload(new DefaultPayload(requestBody)); + JsonPayload samePayload = new JsonPayload(new DefaultPayload(requestBody)); + JsonPayload difPayload = new JsonPayload(new DefaultPayload(difRequestBody)); + + Assertions.assertEquals(payload.hashCode(), samePayload.hashCode()); + Assertions.assertNotEquals(payload.hashCode(), difPayload.hashCode()); } } diff --git a/src/test/java/com/teragrep/lsh_01/PathPropertiesTest.java b/src/test/java/com/teragrep/lsh_01/PathPropertiesTest.java new file mode 100644 index 0000000..94e43ff --- /dev/null +++ b/src/test/java/com/teragrep/lsh_01/PathPropertiesTest.java @@ -0,0 +1,57 @@ +/* + logstash-http-input to syslog bridge + Copyright 2024 Suomen Kanuuna Oy + + Derivative Work of Elasticsearch + Copyright 2012-2015 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.teragrep.lsh_01; + +import com.teragrep.lsh_01.config.PathProperties; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class PathPropertiesTest { + + @Test + public void testEquals() { + String fileName = "src/test/resources/properties/defaultTest.properties"; + PathProperties props1 = new PathProperties(fileName); + PathProperties props2 = new PathProperties(fileName); + + // calling functions shouldn't have effect on an immutable object + Assertions.assertDoesNotThrow(props1::deepCopyAsUnmodifiableMap); + + Assertions.assertEquals(props1, props2); + } + + @Test + public void testNotEquals() { + PathProperties props1 = new PathProperties("src/test/resources/properties/defaultTest.properties"); + PathProperties props2 = new PathProperties("src/test/resources/properties/customTest.properties"); + + Assertions.assertNotEquals(props1, props2); + } + + @Test + public void testHashCode() { + PathProperties props1 = new PathProperties("src/test/resources/properties/defaultTest.properties"); + PathProperties props2 = new PathProperties("src/test/resources/properties/defaultTest.properties"); + PathProperties props3 = new PathProperties("src/test/resources/properties/customTest.properties"); + + Assertions.assertEquals(props1.hashCode(), props2.hashCode()); + Assertions.assertNotEquals(props1.hashCode(), props3.hashCode()); + } +} diff --git a/src/test/java/com/teragrep/lsh_01/RegexPayloadTest.java b/src/test/java/com/teragrep/lsh_01/RegexPayloadTest.java index 6dc734b..d6763b4 100644 --- a/src/test/java/com/teragrep/lsh_01/RegexPayloadTest.java +++ b/src/test/java/com/teragrep/lsh_01/RegexPayloadTest.java @@ -92,4 +92,20 @@ public void testObjectNotEquals() { Assertions.assertNotEquals(payload, difPayload); Assertions.assertNotEquals(payload, difPattern); } + + @Test + public void testHashCode() { + Pattern splitPattern = Pattern.compile("\\n"); + Pattern difSplitPattern = Pattern.compile(","); + String requestBody = "[\n{\"foo\": 1}\n]"; + String difRequestBody = "[\n{\"bar\": 2}\n]"; + RegexPayload payload1 = new RegexPayload(new DefaultPayload(requestBody), splitPattern); + RegexPayload payload2 = new RegexPayload(new DefaultPayload(requestBody), splitPattern); + RegexPayload payload3 = new RegexPayload(new DefaultPayload(difRequestBody), splitPattern); + RegexPayload payload4 = new RegexPayload(new DefaultPayload(requestBody), difSplitPattern); + + Assertions.assertEquals(payload1.hashCode(), payload2.hashCode()); + Assertions.assertNotEquals(payload1.hashCode(), payload3.hashCode()); + Assertions.assertNotEquals(payload1.hashCode(), payload4.hashCode()); + } } diff --git a/src/test/java/com/teragrep/lsh_01/SecurityConfigTest.java b/src/test/java/com/teragrep/lsh_01/SecurityConfigTest.java new file mode 100644 index 0000000..eaf6357 --- /dev/null +++ b/src/test/java/com/teragrep/lsh_01/SecurityConfigTest.java @@ -0,0 +1,64 @@ +/* + logstash-http-input to syslog bridge + Copyright 2024 Suomen Kanuuna Oy + + Derivative Work of Elasticsearch + Copyright 2012-2015 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.teragrep.lsh_01; + +import com.teragrep.lsh_01.config.SecurityConfig; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class SecurityConfigTest { + + @Test + public void testEquals() { + System.setProperty("properties.file", "src/test/resources/properties/defaultTest.properties"); + SecurityConfig config1 = new SecurityConfig(); + SecurityConfig config2 = new SecurityConfig(); + + // calling functions shouldn't have effect on an immutable object + config1.validate(); + + Assertions.assertEquals(config1, config2); + System.clearProperty("properties.file"); + } + + @Test + public void testNotEquals() { + System.setProperty("properties.file", "src/test/resources/properties/defaultTest.properties"); + SecurityConfig config1 = new SecurityConfig(); + System.setProperty("properties.file", "src/test/resources/properties/customTest.properties"); + SecurityConfig config2 = new SecurityConfig(); + + Assertions.assertNotEquals(config1, config2); + System.clearProperty("properties.file"); + } + + @Test + public void testHashCode() { + System.setProperty("properties.file", "src/test/resources/properties/defaultTest.properties"); + SecurityConfig config1 = new SecurityConfig(); + SecurityConfig config2 = new SecurityConfig(); + System.setProperty("properties.file", "src/test/resources/properties/customTest.properties"); + SecurityConfig config3 = new SecurityConfig(); + + Assertions.assertEquals(config1.hashCode(), config2.hashCode()); + Assertions.assertNotEquals(config1.hashCode(), config3.hashCode()); + System.clearProperty("properties.file"); + } +} diff --git a/src/test/resources/exampleCredentials.json b/src/test/resources/exampleCredentials.json new file mode 100644 index 0000000..2862ea9 --- /dev/null +++ b/src/test/resources/exampleCredentials.json @@ -0,0 +1,6 @@ +[ + { + "identity": "ExampleUser", + "credential": "ExamplePassword" + } +] diff --git a/src/test/resources/properties/customTest.properties b/src/test/resources/properties/customTest.properties new file mode 100644 index 0000000..b2498a0 --- /dev/null +++ b/src/test/resources/properties/customTest.properties @@ -0,0 +1,26 @@ +server.listenAddress=127.0.0.1 +server.listenPort=1234 +server.threads=1 +server.maxPendingRequests=128 +server.maxContentLength=262144 + +healthcheck.enabled=true +healthcheck.url=/healthcheck + +relp.target=127.0.0.1 +relp.port=1601 +relp.reconnectInterval=1000 +relp.rebindRequestAmount=1000000 +relp.rebindEnabled=false + +security.authRequired=false + +credentials.file=etc/credentials.json + +lookups.hostname.file=etc/hostname.json +lookups.appname.file=etc/appname.json + +payload.splitType=regex +payload.splitType.regex.pattern=\n + +prometheus.port=1234 diff --git a/src/test/resources/properties/defaultTest.properties b/src/test/resources/properties/defaultTest.properties new file mode 100644 index 0000000..2809003 --- /dev/null +++ b/src/test/resources/properties/defaultTest.properties @@ -0,0 +1,26 @@ +server.listenAddress=127.0.0.1 +server.listenPort=8080 +server.threads=1 +server.maxPendingRequests=128 +server.maxContentLength=262144 + +healthcheck.enabled=true +healthcheck.url=/healthcheck + +relp.target=127.0.0.1 +relp.port=601 +relp.reconnectInterval=10000 +relp.rebindRequestAmount=1000000 +relp.rebindEnabled=false + +security.authRequired=true + +credentials.file=etc/credentials.json + +lookups.hostname.file=etc/hostname.json +lookups.appname.file=etc/appname.json + +payload.splitType=none +payload.splitType.regex.pattern=\n + +prometheus.port=1234 From 7b30235f29454a77407c105edb27d36181812142 Mon Sep 17 00:00:00 2001 From: Rasmus Jokinen <146736881+51-code@users.noreply.github.com> Date: Thu, 10 Oct 2024 14:31:01 +0300 Subject: [PATCH 07/10] Add missing hashCodes to ConversionFactory and LookupConfig --- .../lsh_01/config/ConversionFactory.java | 6 +++ .../teragrep/lsh_01/config/LookupConfig.java | 7 ++++ .../lsh_01/ConversionFactoryTest.java | 41 +++++++++++++++++++ 3 files changed, 54 insertions(+) diff --git a/src/main/java/com/teragrep/lsh_01/config/ConversionFactory.java b/src/main/java/com/teragrep/lsh_01/config/ConversionFactory.java index 2a50d7f..8020993 100644 --- a/src/main/java/com/teragrep/lsh_01/config/ConversionFactory.java +++ b/src/main/java/com/teragrep/lsh_01/config/ConversionFactory.java @@ -30,6 +30,7 @@ import org.apache.logging.log4j.Logger; import java.util.Map; +import java.util.Objects; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; @@ -139,4 +140,9 @@ public boolean equals(Object o) { .equals(cast.pool) && securityConfig.equals(cast.securityConfig) && basicAuthentication.equals(cast.basicAuthentication) && lookupConfig.equals(cast.lookupConfig); } + + @Override + public int hashCode() { + return Objects.hash(splitType, regexPattern, pool, securityConfig, basicAuthentication, lookupConfig); + } } diff --git a/src/main/java/com/teragrep/lsh_01/config/LookupConfig.java b/src/main/java/com/teragrep/lsh_01/config/LookupConfig.java index ca3251c..fabe9a6 100644 --- a/src/main/java/com/teragrep/lsh_01/config/LookupConfig.java +++ b/src/main/java/com/teragrep/lsh_01/config/LookupConfig.java @@ -19,6 +19,8 @@ */ package com.teragrep.lsh_01.config; +import java.util.Objects; + public class LookupConfig implements Validateable { public final String hostnamePath; @@ -50,4 +52,9 @@ public boolean equals(Object o) { final LookupConfig cast = (LookupConfig) o; return hostnamePath.equals(cast.hostnamePath) && appNamePath.equals(cast.appNamePath); } + + @Override + public int hashCode() { + return Objects.hash(hostnamePath, appNamePath); + } } diff --git a/src/test/java/com/teragrep/lsh_01/ConversionFactoryTest.java b/src/test/java/com/teragrep/lsh_01/ConversionFactoryTest.java index f884c9f..da28227 100644 --- a/src/test/java/com/teragrep/lsh_01/ConversionFactoryTest.java +++ b/src/test/java/com/teragrep/lsh_01/ConversionFactoryTest.java @@ -208,4 +208,45 @@ public void testNotEqualConversionFactories() { Assertions.assertNotEquals(conversionFactory, conversionFactoryCopy); } + + @Test + public void testHashCode() { + String regexPattern = ""; + String splitType = "json_array"; + + RelpConnectionFactory relpConnectionFactory = new RelpConnectionFactory(new RelpConfig()); + Pool pool = new Pool<>(relpConnectionFactory, new ManagedRelpConnectionStub()); + BasicAuthentication auth = new BasicAuthenticationFactory().create(); + + ConversionFactory conversionFactory1 = new ConversionFactory( + splitType, + regexPattern, + pool, + new SecurityConfig(), + auth, + new LookupConfig() + ); + ConversionFactory conversionFactory2 = new ConversionFactory( + splitType, + regexPattern, + pool, + new SecurityConfig(), + auth, + new LookupConfig() + ); + + regexPattern = "\n"; + splitType = "regex"; + ConversionFactory conversionFactory3 = new ConversionFactory( + splitType, + regexPattern, + pool, + new SecurityConfig(), + new BasicAuthenticationFactory().create(), + new LookupConfig() + ); + + Assertions.assertEquals(conversionFactory1.hashCode(), conversionFactory2.hashCode()); + Assertions.assertNotEquals(conversionFactory1.hashCode(), conversionFactory3.hashCode()); + } } From 45d05b4a331b5c5aa18fe998100a82294c24e3ce Mon Sep 17 00:00:00 2001 From: Rasmus Jokinen <146736881+51-code@users.noreply.github.com> Date: Thu, 10 Oct 2024 14:55:58 +0300 Subject: [PATCH 08/10] Remove checking System Properties in ConversionFactory, make end-to-end tests use properties from a resource file --- .../lsh_01/config/ConversionFactory.java | 7 ++--- .../teragrep/lsh_01/JsonSplittingTest.java | 8 ++---- .../teragrep/lsh_01/RegexSplittingTest.java | 8 ++---- .../properties/jsonSplittingTest.properties | 26 +++++++++++++++++++ .../properties/regexSplittingTest.properties | 26 +++++++++++++++++++ 5 files changed, 58 insertions(+), 17 deletions(-) create mode 100644 src/test/resources/properties/jsonSplittingTest.properties create mode 100644 src/test/resources/properties/regexSplittingTest.properties diff --git a/src/main/java/com/teragrep/lsh_01/config/ConversionFactory.java b/src/main/java/com/teragrep/lsh_01/config/ConversionFactory.java index 8020993..8d32d55 100644 --- a/src/main/java/com/teragrep/lsh_01/config/ConversionFactory.java +++ b/src/main/java/com/teragrep/lsh_01/config/ConversionFactory.java @@ -54,11 +54,8 @@ public ConversionFactory( ) { // if system property is not specified, defaults to config file (the Map) this( - System.getProperty("payload.splitType", configuration.get("payload.splitType")), - System - .getProperty( - "payload.splitType.regex.pattern", configuration.get("payload.splitType.regex.pattern") - ), + configuration.get("payload.splitType"), + configuration.get("payload.splitType.regex.pattern"), pool, securityConfig, basicAuthentication, diff --git a/src/test/java/com/teragrep/lsh_01/JsonSplittingTest.java b/src/test/java/com/teragrep/lsh_01/JsonSplittingTest.java index 8244dde..3a7837c 100644 --- a/src/test/java/com/teragrep/lsh_01/JsonSplittingTest.java +++ b/src/test/java/com/teragrep/lsh_01/JsonSplittingTest.java @@ -39,9 +39,7 @@ public class JsonSplittingTest { @BeforeAll void setUp() throws InterruptedException { // have to set config here before running Main - System.setProperty("payload.splitType", "json_array"); - System.setProperty("security.authRequired", "false"); - System.setProperty("relp.port", "1601"); + System.setProperty("properties.file", "src/test/resources/properties/jsonSplittingTest.properties"); // Start listening to HTTP-requests program = new Thread(() -> Main.main(new String[] {})); @@ -60,9 +58,7 @@ void reset() { @AfterAll void tearDown() { - System.clearProperty("payload.splitType"); - System.clearProperty("security.authRequired"); - System.clearProperty("relp.port"); + System.clearProperty("properties.file"); this.relpServer.tearDown(); program.interrupt(); } diff --git a/src/test/java/com/teragrep/lsh_01/RegexSplittingTest.java b/src/test/java/com/teragrep/lsh_01/RegexSplittingTest.java index 3ba36bd..2ca206e 100644 --- a/src/test/java/com/teragrep/lsh_01/RegexSplittingTest.java +++ b/src/test/java/com/teragrep/lsh_01/RegexSplittingTest.java @@ -39,9 +39,7 @@ public class RegexSplittingTest { @BeforeAll void setUp() throws InterruptedException { - System.setProperty("payload.splitType", "regex"); - System.setProperty("security.authRequired", "false"); - System.setProperty("relp.port", "1601"); + System.setProperty("properties.file", "src/test/resources/properties/regexSplittingTest.properties"); // Start listening to HTTP-requests program = new Thread(() -> Main.main(new String[] {})); @@ -62,9 +60,7 @@ void reset() { @AfterAll void tearDown() { - System.clearProperty("payload.splitType"); - System.clearProperty("security.authRequired"); - System.clearProperty("relp.port"); + System.clearProperty("properties.file"); this.relpServer.tearDown(); program.interrupt(); } diff --git a/src/test/resources/properties/jsonSplittingTest.properties b/src/test/resources/properties/jsonSplittingTest.properties new file mode 100644 index 0000000..ae78fac --- /dev/null +++ b/src/test/resources/properties/jsonSplittingTest.properties @@ -0,0 +1,26 @@ +server.listenAddress=127.0.0.1 +server.listenPort=8080 +server.threads=1 +server.maxPendingRequests=128 +server.maxContentLength=262144 + +healthcheck.enabled=true +healthcheck.url=/healthcheck + +relp.target=127.0.0.1 +relp.port=1601 +relp.reconnectInterval=10000 +relp.rebindRequestAmount=1000000 +relp.rebindEnabled=false + +security.authRequired=false + +credentials.file=etc/credentials.json + +lookups.hostname.file=etc/hostname.json +lookups.appname.file=etc/appname.json + +payload.splitType=json_array +payload.splitType.regex.pattern=\n + +prometheus.port=1234 \ No newline at end of file diff --git a/src/test/resources/properties/regexSplittingTest.properties b/src/test/resources/properties/regexSplittingTest.properties new file mode 100644 index 0000000..9933e26 --- /dev/null +++ b/src/test/resources/properties/regexSplittingTest.properties @@ -0,0 +1,26 @@ +server.listenAddress=127.0.0.1 +server.listenPort=8080 +server.threads=1 +server.maxPendingRequests=128 +server.maxContentLength=262144 + +healthcheck.enabled=true +healthcheck.url=/healthcheck + +relp.target=127.0.0.1 +relp.port=1601 +relp.reconnectInterval=10000 +relp.rebindRequestAmount=1000000 +relp.rebindEnabled=false + +security.authRequired=false + +credentials.file=etc/credentials.json + +lookups.hostname.file=etc/hostname.json +lookups.appname.file=etc/appname.json + +payload.splitType=regex +payload.splitType.regex.pattern=\n + +prometheus.port=1234 \ No newline at end of file From fd57f5418784adeb6b849c524f9aac3932e510ed Mon Sep 17 00:00:00 2001 From: Rasmus Jokinen <146736881+51-code@users.noreply.github.com> Date: Wed, 16 Oct 2024 09:16:14 +0300 Subject: [PATCH 09/10] Use rlo_06 for asserting payload correctness, add multithreading test --- pom.xml | 7 + .../teragrep/lsh_01/JsonSplittingTest.java | 73 +++++---- .../lsh_01/MultithreadingEndToEndTest.java | 139 ++++++++++++++++++ .../teragrep/lsh_01/RegexSplittingTest.java | 53 +++++-- .../properties/multithreadingTest.properties | 26 ++++ 5 files changed, 260 insertions(+), 38 deletions(-) create mode 100644 src/test/java/com/teragrep/lsh_01/MultithreadingEndToEndTest.java create mode 100644 src/test/resources/properties/multithreadingTest.properties diff --git a/pom.xml b/pom.xml index 14f5dc9..bb71506 100644 --- a/pom.xml +++ b/pom.xml @@ -170,6 +170,13 @@ rlp_03 ${rlp_03.version} + + + com.teragrep + rlo_06 + 9.0.1 + test + org.elasticsearch diff --git a/src/test/java/com/teragrep/lsh_01/JsonSplittingTest.java b/src/test/java/com/teragrep/lsh_01/JsonSplittingTest.java index 3a7837c..56945d4 100644 --- a/src/test/java/com/teragrep/lsh_01/JsonSplittingTest.java +++ b/src/test/java/com/teragrep/lsh_01/JsonSplittingTest.java @@ -21,12 +21,16 @@ import com.teragrep.lsh_01.config.NettyConfig; import com.teragrep.lsh_01.util.RelpServer; +import com.teragrep.rlo_06.RFC5424Frame; import org.junit.jupiter.api.*; +import java.io.ByteArrayInputStream; import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; @@ -67,6 +71,7 @@ void tearDown() { public void testJsonSplittingOneMessage() { // no splitting needed String message = "{\"foo\": 1}"; String requestBody = "[\n" + message + "\n]"; + String expected = message.replaceAll("\\s", ""); NettyConfig nettyConfig = new NettyConfig(); @@ -85,17 +90,25 @@ public void testJsonSplittingOneMessage() { // no splitting needed List payloads = this.relpServer.payloads(); - String expected = message.replaceAll("\\s", ""); - // assert that there is just one payload with the correct message Assertions.assertEquals(1, payloads.size()); - Assertions.assertTrue(payloads.get(0).contains(expected)); + + RFC5424Frame frame = new RFC5424Frame(); + frame.load(new ByteArrayInputStream(payloads.get(0).getBytes(StandardCharsets.UTF_8))); + Assertions.assertDoesNotThrow(frame::next); + + Assertions.assertEquals(expected, frame.msg.toString()); } @Test public void testJsonSplittingTwoMessages() { String message1 = "{\"foo\": 1}"; String message2 = "{\"bar\": 2}"; + + ArrayList expectedList = new ArrayList<>(); + expectedList.add(message1.replaceAll("\\s", "")); + expectedList.add(message2.replaceAll("\\s", "")); + String requestBody = "[\n" + message1 + ",\n" + message2 + "\n]"; NettyConfig nettyConfig = new NettyConfig(); @@ -115,16 +128,18 @@ public void testJsonSplittingTwoMessages() { List payloads = this.relpServer.payloads(); - String expected1 = message1.replaceAll("\\s", ""); - String expected2 = message2.replaceAll("\\s", ""); - // assert that payload was correctly split Assertions.assertEquals(2, payloads.size()); - Assertions.assertTrue(payloads.get(0).contains(expected1)); - Assertions.assertFalse(payloads.get(0).contains(expected2)); - Assertions.assertTrue(payloads.get(1).contains(expected2)); - Assertions.assertFalse(payloads.get(1).contains(expected1)); + int loops = 0; + RFC5424Frame frame = new RFC5424Frame(); + for (int i = 0; i < payloads.size(); i++) { + frame.load(new ByteArrayInputStream(payloads.get(i).getBytes(StandardCharsets.UTF_8))); + Assertions.assertDoesNotThrow(frame::next); + Assertions.assertEquals(expectedList.get(i), frame.msg.toString()); + loops++; + } + Assertions.assertEquals(payloads.size(), loops); } @Test @@ -134,6 +149,11 @@ public void testJsonSplittingThreeMessages() { String message3 = "{\"foobar\": 3}"; String requestBody = "[\n" + message1 + ",\n" + message2 + "\n, \n" + message3 + "\n]"; + ArrayList expectedList = new ArrayList<>(); + expectedList.add(message1.replaceAll("\\s", "")); + expectedList.add(message2.replaceAll("\\s", "")); + expectedList.add(message3.replaceAll("\\s", "")); + NettyConfig nettyConfig = new NettyConfig(); HttpClient httpClient = HttpClient.newHttpClient(); @@ -151,28 +171,24 @@ public void testJsonSplittingThreeMessages() { List payloads = this.relpServer.payloads(); - String expected1 = message1.replaceAll("\\s", ""); - String expected2 = message2.replaceAll("\\s", ""); - String expected3 = message3.replaceAll("\\s", ""); - // assert that payload was correctly split Assertions.assertEquals(3, payloads.size()); - Assertions.assertTrue(payloads.get(0).contains(expected1)); - Assertions.assertFalse(payloads.get(0).contains(expected2)); - Assertions.assertFalse(payloads.get(0).contains(expected3)); - Assertions.assertTrue(payloads.get(1).contains(expected2)); - Assertions.assertFalse(payloads.get(1).contains(expected1)); - Assertions.assertFalse(payloads.get(1).contains(expected3)); - - Assertions.assertTrue(payloads.get(2).contains(expected3)); - Assertions.assertFalse(payloads.get(2).contains(expected1)); - Assertions.assertFalse(payloads.get(2).contains(expected2)); + int loops = 0; + RFC5424Frame frame = new RFC5424Frame(); + for (int i = 0; i < payloads.size(); i++) { + frame.load(new ByteArrayInputStream(payloads.get(i).getBytes(StandardCharsets.UTF_8))); + Assertions.assertDoesNotThrow(frame::next); + Assertions.assertEquals(expectedList.get(i), frame.msg.toString()); + loops++; + } + Assertions.assertEquals(payloads.size(), loops); } @Test public void testJsonSplittingNestedObjects() { String payload = "{\"foo\": {\"bar\": 2}}"; + String expected = payload.replaceAll("\\s", ""); String requestBody = "[\n" + payload + "\n]"; NettyConfig nettyConfig = new NettyConfig(); @@ -192,10 +208,13 @@ public void testJsonSplittingNestedObjects() { List payloads = this.relpServer.payloads(); - String expected = payload.replaceAll("\\s", ""); - // assert that payload was correctly split Assertions.assertEquals(1, payloads.size()); - Assertions.assertTrue(payloads.get(0).contains(expected)); + + RFC5424Frame frame = new RFC5424Frame(); + frame.load(new ByteArrayInputStream(payloads.get(0).getBytes(StandardCharsets.UTF_8))); + Assertions.assertDoesNotThrow(frame::next); + + Assertions.assertEquals(expected, frame.msg.toString()); } } diff --git a/src/test/java/com/teragrep/lsh_01/MultithreadingEndToEndTest.java b/src/test/java/com/teragrep/lsh_01/MultithreadingEndToEndTest.java new file mode 100644 index 0000000..04b4098 --- /dev/null +++ b/src/test/java/com/teragrep/lsh_01/MultithreadingEndToEndTest.java @@ -0,0 +1,139 @@ +/* + logstash-http-input to syslog bridge + Copyright 2024 Suomen Kanuuna Oy + + Derivative Work of Elasticsearch + Copyright 2012-2015 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.teragrep.lsh_01; + +import com.teragrep.lsh_01.config.NettyConfig; +import com.teragrep.lsh_01.util.RelpServer; +import com.teragrep.rlo_06.RFC5424Frame; +import org.junit.jupiter.api.*; + +import java.io.ByteArrayInputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Random; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class MultithreadingEndToEndTest { + + private RelpServer relpServer; + private Thread program; + + @BeforeAll + void setUp() throws InterruptedException { + // have to set config here before running Main + System.setProperty("properties.file", "src/test/resources/properties/multithreadingTest.properties"); + + // Start listening to HTTP-requests + program = new Thread(() -> Main.main(new String[] {})); + program.start(); + + Thread.sleep(3000); // wait for netty to start up + + this.relpServer = new RelpServer(); + this.relpServer.setUpDefault(); + } + + @AfterEach + void reset() { + this.relpServer.clear(); + } + + @AfterAll + void tearDown() { + System.clearProperty("properties.file"); + this.relpServer.tearDown(); + program.interrupt(); + } + + @Test + public void testMultithreadingTenClients() { + List requestBodies = Collections.synchronizedList(new ArrayList<>()); + + HttpClient httpClient = HttpClient.newHttpClient(); + NettyConfig nettyConfig = new NettyConfig(); + + final int clients = 10; + final int messagesPerClient = 100; + + ExecutorService executor = Executors.newFixedThreadPool(clients); + List> futures = new ArrayList<>(); + + // Send messages in parallel + for (int i = 0; i < clients; i++) { + Future future = executor.submit(() -> { + for (int j = 0; j < messagesPerClient; j++) { + String requestBody = randomString(); + requestBodies.add(requestBody); + + HttpRequest request = HttpRequest + .newBuilder(URI.create("http://" + nettyConfig.listenAddress + ":" + nettyConfig.listenPort)).POST(HttpRequest.BodyPublishers.ofString(requestBody)).build(); + + CompletableFuture> response = httpClient + .sendAsync(request, HttpResponse.BodyHandlers.ofString()); + + // Assert that there is a successful response + int statusCode = Assertions.assertDoesNotThrow(() -> response.get().statusCode()); + Assertions.assertEquals(200, statusCode); + } + }); + futures.add(future); + } + + // wait until all threads are done + for (Future future : futures) + Assertions.assertDoesNotThrow(() -> future.get()); + + List payloads = this.relpServer.payloads(); // get the results + Assertions.assertEquals(clients * messagesPerClient, payloads.size()); + + int loops = 0; + RFC5424Frame frame = new RFC5424Frame(); + for (String payload : payloads) { + frame.load(new ByteArrayInputStream(payload.getBytes(StandardCharsets.UTF_8))); + Assertions.assertDoesNotThrow(frame::next); + Assertions.assertTrue(requestBodies.contains(frame.msg.toString())); // order of payloads can differ + loops++; + } + Assertions.assertEquals(payloads.size(), loops); + } + + private String randomString() { + int leftLimit = 97; // letter 'a' + int rightLimit = 122; // letter 'z' + int targetStringLength = 10; + Random random = new Random(); + + return random + .ints(leftLimit, rightLimit + 1) + .limit(targetStringLength) + .collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append) + .toString(); + } +} diff --git a/src/test/java/com/teragrep/lsh_01/RegexSplittingTest.java b/src/test/java/com/teragrep/lsh_01/RegexSplittingTest.java index 2ca206e..49e4bc5 100644 --- a/src/test/java/com/teragrep/lsh_01/RegexSplittingTest.java +++ b/src/test/java/com/teragrep/lsh_01/RegexSplittingTest.java @@ -21,12 +21,16 @@ import com.teragrep.lsh_01.config.NettyConfig; import com.teragrep.lsh_01.util.RelpServer; +import com.teragrep.rlo_06.RFC5424Frame; import org.junit.jupiter.api.*; +import java.io.ByteArrayInputStream; import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; @@ -67,7 +71,14 @@ void tearDown() { @Test public void testRegexSplittingTwoMessages() { - String requestBody = "foofoo\nbar"; + String expected1 = "foofoo"; + String expected2 = "bar"; + + ArrayList expectedList = new ArrayList<>(); + expectedList.add(expected1); + expectedList.add(expected2); + + String requestBody = expected1 + "\n" + expected2; HttpClient httpClient = HttpClient.newHttpClient(); @@ -86,15 +97,30 @@ public void testRegexSplittingTwoMessages() { // assert that payload was correctly split into two Assertions.assertEquals(2, payloads.size()); - Assertions.assertTrue(payloads.get(0).contains("foofoo")); - Assertions.assertFalse(payloads.get(0).contains("bar")); - Assertions.assertTrue(payloads.get(1).contains("bar")); - Assertions.assertFalse(payloads.get(1).contains("foofoo")); + + int loops = 0; + RFC5424Frame frame = new RFC5424Frame(); + for (int i = 0; i < payloads.size(); i++) { + frame.load(new ByteArrayInputStream(payloads.get(i).getBytes(StandardCharsets.UTF_8))); + Assertions.assertDoesNotThrow(frame::next); + Assertions.assertEquals(expectedList.get(i), frame.msg.toString()); + loops++; + } + Assertions.assertEquals(payloads.size(), loops); } @Test public void testRegexSplittingThreeMessages() { - String requestBody = "foofoo\nbar\nfoo bar"; + String expected1 = "foofoo"; + String expected2 = "bar"; + String expected3 = "foo bar"; + + ArrayList expectedList = new ArrayList<>(); + expectedList.add(expected1); + expectedList.add(expected2); + expectedList.add(expected3); + + String requestBody = expected1 + "\n" + expected2 + "\n" + expected3; HttpClient httpClient = HttpClient.newHttpClient(); @@ -113,10 +139,15 @@ public void testRegexSplittingThreeMessages() { // assert that payload was correctly split into three parts Assertions.assertEquals(3, payloads.size()); - Assertions.assertTrue(payloads.get(0).contains("foofoo")); - Assertions.assertFalse(payloads.get(0).contains("bar")); - Assertions.assertTrue(payloads.get(1).contains("bar")); - Assertions.assertFalse(payloads.get(1).contains("foofoo")); - Assertions.assertTrue(payloads.get(2).contains("foo bar")); + + int loops = 0; + RFC5424Frame frame = new RFC5424Frame(); + for (int i = 0; i < payloads.size(); i++) { + frame.load(new ByteArrayInputStream(payloads.get(i).getBytes(StandardCharsets.UTF_8))); + Assertions.assertDoesNotThrow(frame::next); + Assertions.assertEquals(expectedList.get(i), frame.msg.toString()); + loops++; + } + Assertions.assertEquals(payloads.size(), loops); } } diff --git a/src/test/resources/properties/multithreadingTest.properties b/src/test/resources/properties/multithreadingTest.properties new file mode 100644 index 0000000..977ea36 --- /dev/null +++ b/src/test/resources/properties/multithreadingTest.properties @@ -0,0 +1,26 @@ +server.listenAddress=127.0.0.1 +server.listenPort=8080 +server.threads=10 +server.maxPendingRequests=128 +server.maxContentLength=262144 + +healthcheck.enabled=true +healthcheck.url=/healthcheck + +relp.target=127.0.0.1 +relp.port=1601 +relp.reconnectInterval=10000 +relp.rebindRequestAmount=1000000 +relp.rebindEnabled=false + +security.authRequired=false + +credentials.file=etc/credentials.json + +lookups.hostname.file=etc/hostname.json +lookups.appname.file=etc/appname.json + +payload.splitType=none +payload.splitType.regex.pattern=\n + +prometheus.port=1234 From 5bb31cddf92c317ef36b646b09bcfab7b52a09eb Mon Sep 17 00:00:00 2001 From: Rasmus Jokinen <146736881+51-code@users.noreply.github.com> Date: Wed, 16 Oct 2024 12:40:38 +0300 Subject: [PATCH 10/10] Use expected message amounts instead of result list size as an expected value in assertions --- src/test/java/com/teragrep/lsh_01/JsonSplittingTest.java | 8 ++++---- .../com/teragrep/lsh_01/MultithreadingEndToEndTest.java | 2 +- src/test/java/com/teragrep/lsh_01/RegexSplittingTest.java | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/test/java/com/teragrep/lsh_01/JsonSplittingTest.java b/src/test/java/com/teragrep/lsh_01/JsonSplittingTest.java index 56945d4..081abaa 100644 --- a/src/test/java/com/teragrep/lsh_01/JsonSplittingTest.java +++ b/src/test/java/com/teragrep/lsh_01/JsonSplittingTest.java @@ -129,7 +129,7 @@ public void testJsonSplittingTwoMessages() { List payloads = this.relpServer.payloads(); // assert that payload was correctly split - Assertions.assertEquals(2, payloads.size()); + Assertions.assertEquals(expectedList.size(), payloads.size()); int loops = 0; RFC5424Frame frame = new RFC5424Frame(); @@ -139,7 +139,7 @@ public void testJsonSplittingTwoMessages() { Assertions.assertEquals(expectedList.get(i), frame.msg.toString()); loops++; } - Assertions.assertEquals(payloads.size(), loops); + Assertions.assertEquals(expectedList.size(), loops); } @Test @@ -172,7 +172,7 @@ public void testJsonSplittingThreeMessages() { List payloads = this.relpServer.payloads(); // assert that payload was correctly split - Assertions.assertEquals(3, payloads.size()); + Assertions.assertEquals(expectedList.size(), payloads.size()); int loops = 0; RFC5424Frame frame = new RFC5424Frame(); @@ -182,7 +182,7 @@ public void testJsonSplittingThreeMessages() { Assertions.assertEquals(expectedList.get(i), frame.msg.toString()); loops++; } - Assertions.assertEquals(payloads.size(), loops); + Assertions.assertEquals(expectedList.size(), loops); } @Test diff --git a/src/test/java/com/teragrep/lsh_01/MultithreadingEndToEndTest.java b/src/test/java/com/teragrep/lsh_01/MultithreadingEndToEndTest.java index 04b4098..ee226c1 100644 --- a/src/test/java/com/teragrep/lsh_01/MultithreadingEndToEndTest.java +++ b/src/test/java/com/teragrep/lsh_01/MultithreadingEndToEndTest.java @@ -121,7 +121,7 @@ public void testMultithreadingTenClients() { Assertions.assertTrue(requestBodies.contains(frame.msg.toString())); // order of payloads can differ loops++; } - Assertions.assertEquals(payloads.size(), loops); + Assertions.assertEquals(clients * messagesPerClient, loops); } private String randomString() { diff --git a/src/test/java/com/teragrep/lsh_01/RegexSplittingTest.java b/src/test/java/com/teragrep/lsh_01/RegexSplittingTest.java index 49e4bc5..8829d43 100644 --- a/src/test/java/com/teragrep/lsh_01/RegexSplittingTest.java +++ b/src/test/java/com/teragrep/lsh_01/RegexSplittingTest.java @@ -96,7 +96,7 @@ public void testRegexSplittingTwoMessages() { List payloads = this.relpServer.payloads(); // assert that payload was correctly split into two - Assertions.assertEquals(2, payloads.size()); + Assertions.assertEquals(expectedList.size(), payloads.size()); int loops = 0; RFC5424Frame frame = new RFC5424Frame(); @@ -106,7 +106,7 @@ public void testRegexSplittingTwoMessages() { Assertions.assertEquals(expectedList.get(i), frame.msg.toString()); loops++; } - Assertions.assertEquals(payloads.size(), loops); + Assertions.assertEquals(expectedList.size(), loops); } @Test @@ -138,7 +138,7 @@ public void testRegexSplittingThreeMessages() { List payloads = this.relpServer.payloads(); // assert that payload was correctly split into three parts - Assertions.assertEquals(3, payloads.size()); + Assertions.assertEquals(expectedList.size(), payloads.size()); int loops = 0; RFC5424Frame frame = new RFC5424Frame(); @@ -148,6 +148,6 @@ public void testRegexSplittingThreeMessages() { Assertions.assertEquals(expectedList.get(i), frame.msg.toString()); loops++; } - Assertions.assertEquals(payloads.size(), loops); + Assertions.assertEquals(expectedList.size(), loops); } }