diff --git a/archetype/src/main/resources/archetype-resources/crawler-conf.yaml b/archetype/src/main/resources/archetype-resources/crawler-conf.yaml index bc57e3714..6db628282 100644 --- a/archetype/src/main/resources/archetype-resources/crawler-conf.yaml +++ b/archetype/src/main/resources/archetype-resources/crawler-conf.yaml @@ -72,6 +72,12 @@ config: http.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol" https.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol" + # Accept any TLS certificate, e.g. self-signed ones? The servers are not + # authenticated then; credentials (basic auth, credential headers, cookies) + # are withheld over such connections unless + # http.credentials.allow.insecure is set to true. + http.trust.everything: false + # The maximum number of bytes for returned HTTP response bodies. # The fetched page will be trimmed to 65KB in this case # Set -1 to disable the limit. diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java index 927508b43..7717be59c 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java @@ -28,18 +28,18 @@ import java.security.cert.CertificateException; import java.util.ArrayList; import java.util.Base64; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import javax.net.ssl.HostnameVerifier; import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLSession; import javax.net.ssl.SSLSocketFactory; import javax.net.ssl.TrustManager; import javax.net.ssl.X509TrustManager; @@ -54,6 +54,7 @@ import okhttp3.Gzip; import okhttp3.Handshake; import okhttp3.Headers; +import okhttp3.HttpUrl; import okhttp3.Interceptor; import okhttp3.MediaType; import okhttp3.OkHttpClient; @@ -108,6 +109,41 @@ public class HttpProtocol extends AbstractHttpProtocol { // makes sure that a missing cookie origin is reported once and not for every url private final AtomicBoolean missingCookieOriginLogged = new AtomicBoolean(); + // makes sure that withheld cookies are reported once and not for every url + private final AtomicBoolean withheldCookiesLogged = new AtomicBoolean(); + + // makes sure that withheld request headers are reported once and not for every url + private final AtomicBoolean withheldRequestHeadersLogged = new AtomicBoolean(); + + /** Default for http.credentials.headers: header names (lower case) carrying credentials. */ + private static final Set DEFAULT_CREDENTIAL_HEADERS = + Set.of( + HttpHeaders.AUTHORIZATION.toLowerCase(Locale.ROOT), + HttpHeaders.PROXY_AUTHORIZATION.toLowerCase(Locale.ROOT), + // the cookie header is not a constant in HttpHeaders + "cookie", + "x-api-key"); + + // lower case header names considered to carry credentials + private Set credentialHeaders = DEFAULT_CREDENTIAL_HEADERS; + + // request headers withheld from servers which were not authenticated + private final List credentialRequestHeaders = new LinkedList<>(); + + // http.trust.everything: accept any certificate chain + private boolean trustEverything = false; + + // http.verify.hostnames: check the certificate against the host name contacted + private boolean verifyHostnames = true; + + // http.credentials.allow.insecure + private boolean insecureCredentialsAllowed = false; + + /** Tags a request whose Proxy-Authorization was set by the proxy authenticator. */ + private enum ProxyAuthenticated { + INSTANCE + } + private OkHttpClient.Builder builder; private static final TrustManager[] trustAllCerts = @@ -130,11 +166,12 @@ public java.security.cert.X509Certificate[] getAcceptedIssuers() { } }; - private static final SSLContext trustAllSslContext; + // package-private so that the OkHttpTrustEverythingTest can check the protocol + static final SSLContext trustAllSslContext; static { try { - trustAllSslContext = SSLContext.getInstance("SSL"); + trustAllSslContext = SSLContext.getInstance("TLS"); trustAllSslContext.init(null, trustAllCerts, new java.security.SecureRandom()); } catch (Exception e) { throw new RuntimeException(e); @@ -158,6 +195,36 @@ public void configure(Config conf) { this.partialContentAsTrimmed = ConfUtils.getBoolean(conf, "http.content.partial.as.trimmed", false); + this.trustEverything = ConfUtils.getBoolean(conf, "http.trust.everything", false); + this.verifyHostnames = ConfUtils.getBoolean(conf, "http.verify.hostnames", true); + this.insecureCredentialsAllowed = + ConfUtils.getBoolean(conf, "http.credentials.allow.insecure", false); + + // http.credentials.headers adds names to the built-in ones, it cannot remove them + final List configuredCredentialHeaders = + ConfUtils.loadListFromConf("http.credentials.headers", conf); + if (!configuredCredentialHeaders.isEmpty()) { + final Set names = new HashSet<>(DEFAULT_CREDENTIAL_HEADERS); + for (String name : configuredCredentialHeaders) { + if (StringUtils.isNotBlank(name)) { + names.add(name.trim().toLowerCase(Locale.ROOT)); + } + } + credentialHeaders = names; + } + + if (trustEverything) { + LOG.warn( + "http.trust.everything is enabled: TLS certificate chains are accepted without " + + "validation, the identity of the servers is not authenticated. Anybody " + + "able to answer for the host name receives everything sent to them."); + } + if (!verifyHostnames) { + LOG.warn( + "http.verify.hostnames is disabled: the certificates are not checked against " + + "the host name either, the identity of the servers is not authenticated."); + } + builder = new OkHttpClient.Builder() .retryOnConnectionFailure( @@ -220,7 +287,7 @@ public void configure(Config conf) { final String basicAuthUser = ConfUtils.getString(conf, "http.basicauth.user", null); - // use a basic auth? + // use a basic auth? the header is withheld unless the server was authenticated if (StringUtils.isNotBlank(basicAuthUser)) { final String basicAuthPass = ConfUtils.getString(conf, "http.basicauth.password", ""); final String encoding = @@ -228,10 +295,35 @@ public void configure(Config conf) { .encodeToString( (basicAuthUser + ":" + basicAuthPass) .getBytes(StandardCharsets.UTF_8)); - customRequestHeaders.add(new KeyValue(HttpHeaders.AUTHORIZATION, "Basic " + encoding)); + credentialRequestHeaders.add( + new KeyValue(HttpHeaders.AUTHORIZATION, "Basic " + encoding)); } - customHeaders.forEach(customRequestHeaders::add); + for (KeyValue customHeader : customHeaders) { + if (isCredentialHeader(customHeader.getKey())) { + credentialRequestHeaders.add(customHeader); + } else { + customRequestHeaders.add(customHeader); + } + } + + if (!credentialRequestHeaders.isEmpty() || useCookies) { + if (trustEverything && !insecureCredentialsAllowed) { + LOG.warn( + "Credentials configured with http.basicauth.*, credential headers in " + + "http.custom.headers and cookies are withheld from every request " + + "because the servers are not authenticated (http.trust.everything). " + + "Set http.credentials.allow.insecure to true to send them anyway."); + } else if (!insecureCredentialsAllowed) { + LOG.info( + "Credentials configured with http.basicauth.*, credential headers in " + + "http.custom.headers and cookies are sent only over https:// urls " + + "with certificate validation. They are withheld from cleartext " + + "http:// urls and from https:// urls when http.trust.everything " + + "is enabled; set http.credentials.allow.insecure to true to send " + + "them there anyway."); + } + } // optionally block connections to forbidden IP address ranges // (e.g. localhost/loopback, private/site-local addresses), see @@ -241,19 +333,41 @@ public void configure(Config conf) { builder.addNetworkInterceptor(new HTTPFilterIPAddressInterceptor(ipFilterRules)); } + // getProtocolOutput only filters the initial request, the redirect follower copies + // the headers onto the next hop without re-checking. Registered before the + // HTTPHeadersInterceptor so that the request headers it records are the ones sent. + builder.addNetworkInterceptor( + chain -> { + Request hop = chain.request(); + if (!credentialsAllowed(hop.url())) { + // the proxy authenticator's header is read by the proxy on a cleartext + // hop, anywhere else Proxy-Authorization reaches the crawled server + final boolean forProxy = + hop.tag(ProxyAuthenticated.class) != null && !hop.url().isHttps(); + Request.Builder stripped = hop.newBuilder(); + for (String name : new HashSet<>(hop.headers().names())) { + if (forProxy + && HttpHeaders.PROXY_AUTHORIZATION.equalsIgnoreCase(name)) { + continue; + } + if (isCredentialHeader(name)) { + stripped.removeHeader(name); + } + } + hop = stripped.build(); + } + return chain.proceed(hop); + }); + if (storeHttpHeaders) { builder.addNetworkInterceptor(new HTTPHeadersInterceptor()); } - if (ConfUtils.getBoolean(conf, "http.trust.everything", true)) { + if (trustEverything) { builder.sslSocketFactory(trustAllSslSocketFactory, (X509TrustManager) trustAllCerts[0]); - builder.hostnameVerifier( - new HostnameVerifier() { - @Override - public boolean verify(String hostname, SSLSession session) { - return true; - } - }); + } + if (!verifyHostnames) { + builder.hostnameVerifier((hostname, session) -> true); } builder.eventListenerFactory( @@ -284,12 +398,20 @@ public EventListener create(Call call) { client = builder.build(); } - private void addCookiesToRequest(Builder rb, String url, Metadata md) { + private void addCookiesToRequest(Builder rb, String url, Metadata md, boolean sendCredentials) { final String[] cookieStrings = md.getValues(RESPONSE_COOKIES_HEADER, protocolMetadataPrefix); if (cookieStrings == null || cookieStrings.length == 0) { return; } + if (!sendCredentials) { + if (withheldCookiesLogged.compareAndSet(false, true)) { + LOG.warn( + "Cookies are withheld because the server is not authenticated. Set " + + "http.credentials.allow.insecure to true to send them anyway."); + } + return; + } try { final List cookies = CookieConverter.getCookies( @@ -337,12 +459,49 @@ private URL getCookieOrigin(Metadata md, String url) { } } - protected void addHeadersToRequest(Builder rb, Metadata md) { + /** Whether the header carries credentials, see http.credentials.headers. */ + private boolean isCredentialHeader(String name) { + if (name == null) { + return false; + } + final String normalised = name.trim().toLowerCase(Locale.ROOT); + return credentialHeaders.contains(normalised); + } + + /** + * Whether credentials may be sent to the url. The server is only authenticated over https with + * both the certificate chain and the host name checked; http.credentials.allow.insecure sends + * them anyway. + */ + // package-private for the decision matrix in OkHttpTrustEverythingTest + boolean credentialsAllowed(String url) { + if (insecureCredentialsAllowed) { + return true; + } + final HttpUrl parsed = HttpUrl.parse(url); + return parsed != null && credentialsAllowed(parsed); + } + + private boolean credentialsAllowed(HttpUrl url) { + return insecureCredentialsAllowed || (url.isHttps() && !trustEverything && verifyHostnames); + } + + protected void addHeadersToRequest(Builder rb, Metadata md, boolean sendCredentials) { final String[] headerStrings = md.getValues(SET_HEADER_BY_REQUEST, protocolMetadataPrefix); if (headerStrings != null && headerStrings.length > 0) { for (String hs : headerStrings) { KeyValue h = KeyValue.build(hs); + if (!sendCredentials && isCredentialHeader(h.getKey())) { + if (withheldRequestHeadersLogged.compareAndSet(false, true)) { + LOG.warn( + "Header {} set by request is withheld because the server is not " + + "authenticated. Set http.credentials.allow.insecure to " + + "true to send it anyway.", + h.getKey()); + } + continue; + } rb.addHeader(h.getKey(), h.getValue()); } } @@ -379,7 +538,8 @@ public ProtocolResponse getProtocolOutput(String url, final Metadata metadata) // conditionally add proxy authentication if (StringUtils.isNotBlank(prox.getUsername())) { - // add proxy authentication header to builder + // authenticates against the proxy, not the crawled server, + // so it is sent whatever credentialsAllowed decides localBuilder.proxyAuthenticator( (Route route, Response response) -> { String credential = @@ -388,6 +548,9 @@ public ProtocolResponse getProtocolOutput(String url, final Metadata metadata) return response.request() .newBuilder() .header(HttpHeaders.PROXY_AUTHORIZATION, credential) + .tag( + ProxyAuthenticated.class, + ProxyAuthenticated.INSTANCE) .build(); }); } @@ -407,16 +570,33 @@ public ProtocolResponse getProtocolOutput(String url, final Metadata metadata) } } + final boolean sendCredentials = credentialsAllowed(url); + final Builder rb = new Request.Builder().url(url); customRequestHeaders.forEach( (k) -> { rb.header(k.getKey(), k.getValue()); }); + if (sendCredentials) { + credentialRequestHeaders.forEach( + (k) -> { + rb.header(k.getKey(), k.getValue()); + }); + } else if (!credentialRequestHeaders.isEmpty() + && withheldRequestHeadersLogged.compareAndSet(false, true)) { + LOG.warn( + "Configured credential headers (http.basicauth.*, http.custom.headers) are " + + "withheld because {} is not authenticated. Set " + + "http.credentials.allow.insecure to true to send them anyway.", + url.startsWith("https") + ? "https with http.trust.everything or without http.verify.hostnames" + : "cleartext http"); + } int pageMaxContent = globalMaxContent; if (metadata != null) { - addHeadersToRequest(rb, metadata); + addHeadersToRequest(rb, metadata, sendCredentials); final String lastModified = metadata.getFirstValue(HttpHeaders.LAST_MODIFIED); if (StringUtils.isNotBlank(lastModified)) { @@ -453,7 +633,7 @@ public ProtocolResponse getProtocolOutput(String url, final Metadata metadata) } if (useCookies) { - addCookiesToRequest(rb, url, metadata); + addCookiesToRequest(rb, url, metadata, sendCredentials); } final String postJsonData = metadata.getFirstValue("http.post.json"); diff --git a/core/src/main/resources/crawler-default.yaml b/core/src/main/resources/crawler-default.yaml index 2e251dd5e..f20418203 100644 --- a/core/src/main/resources/crawler-default.yaml +++ b/core/src/main/resources/crawler-default.yaml @@ -149,6 +149,45 @@ config: # Follow redirect HTTP responses: http.allow.redirects: false + # Accept any TLS certificate, including self-signed, expired or otherwise + # invalid ones (okhttp protocol only)? The certificate chains are accepted + # without validation, i.e. the servers are not authenticated: anyone able to + # answer for the host name receives everything sent to them. Needed for + # crawling hosts with unvalidatable certificates, e.g. self-signed ones, but + # also disables protection against man-in-the-middle attacks. Credentials + # (basic auth, credential headers, cookies) are withheld from cleartext + # http:// requests and from https:// requests on such connections unless + # http.credentials.allow.insecure is set to true. + http.trust.everything: false + + # Check that the certificate presented by the server matches the host name + # contacted (okhttp protocol only)? Disabling this is independent of + # http.trust.everything: a valid certificate for a different name is then + # accepted. Hostname verification does not need to be disabled for hosts + # with self-signed certificates when http.trust.everything is enabled. When + # disabled, credentials are withheld unless http.credentials.allow.insecure + # is set to true. + http.verify.hostnames: true + + # Send credentials (basic auth, credential headers, cookies) even on + # connections which do not authenticate the server (okhttp protocol only)? + # Without it, credentials are withheld from cleartext http:// requests and + # from https:// requests whose certificate was not validated + # (http.trust.everything) or not checked against the host name + # (http.verify.hostnames: false), so that they are not disclosed to servers + # which were never authenticated. + http.credentials.allow.insecure: false + + # Additional header names (case-insensitive) which carry credentials and are + # therefore withheld from requests to servers which were not authenticated + # (okhttp protocol only, see http.trust.everything and + # http.credentials.allow.insecure). Applies to the headers configured with + # http.custom.headers and to headers set by request; cookies are withheld + # separately. "authorization", "proxy-authorization", "cookie" and + # "x-api-key" are always treated as credentials and cannot be removed here. + # http.credentials.headers: + # - "x-auth-token" + # IP address filtering (okhttp protocol only). Optionally limit or block # connections to IP address ranges once the host name has been resolved. This # prevents information leakage to a public index when a DNS entry points to a diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserRedirectTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserRedirectTest.java index 034489d65..19d7527ce 100644 --- a/core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserRedirectTest.java +++ b/core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserRedirectTest.java @@ -288,6 +288,9 @@ void testHeadersNotSentToAnUnfollowedRedirectTarget() { authConf.putAll(conf); authConf.put("http.basicauth.user", "this_is_only_a_test"); authConf.put("http.basicauth.password", "this_is_only_a_test"); + // the test verifies where the Authorization header is sent, not the + // withholding on unauthenticated connections: opt in + authConf.put("http.credentials.allow.insecure", true); HttpProtocol authProtocol = new HttpProtocol(); authProtocol.configure(authConf); try { diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocolCookieTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocolCookieTest.java index a30d52480..231d8bc2a 100644 --- a/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocolCookieTest.java +++ b/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocolCookieTest.java @@ -148,6 +148,9 @@ private HttpProtocol protocol(boolean useCookies) { conf.put("http.agent.url", "http://test.example.com"); conf.put("http.agent.email", "test@example.com"); conf.put("http.use.cookies", useCookies); + // the tests below verify cookie scoping mechanics, not the withholding + // of credentials on unauthenticated connections: opt in + conf.put("http.credentials.allow.insecure", true); conf.put("protocol.md.prefix", "protocol."); final HttpProtocol protocol = new HttpProtocol(); protocol.configure(conf); diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java new file mode 100644 index 000000000..814ec6f90 --- /dev/null +++ b/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java @@ -0,0 +1,631 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.stormcrawler.protocol.okhttp; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.any; +import static com.github.tomakehurst.wiremock.client.WireMock.anyUrl; +import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.ok; +import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.common.ConsoleNotifier; +import com.github.tomakehurst.wiremock.core.WireMockConfiguration; +import java.io.InputStream; +import java.lang.reflect.Field; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.security.KeyStore; +import java.security.SecureRandom; +import java.util.Base64; +import java.util.List; +import java.util.Locale; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLHandshakeException; +import javax.net.ssl.SSLPeerUnverifiedException; +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; +import javax.net.ssl.X509TrustManager; +import okhttp3.OkHttpClient; +import org.apache.storm.Config; +import org.apache.stormcrawler.Metadata; +import org.apache.stormcrawler.protocol.ProtocolResponse; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +/** + * Regression tests for https://github.com/apache/stormcrawler/issues/2079: the trust-all + * configuration of the okhttp protocol must be an explicit choice, must not disable hostname + * verification and must not disclose credentials to servers which are not authenticated. + */ +/* + * The keystores under src/test/resources/ssl are self-signed PKCS12 keystores, generated with + * (JDK keytool, passwords are the "changeit" default, valid for 9000 days): + * + * keytool -genkeypair -alias localhost -keyalg RSA -keysize 2048 -validity 9000 \ + * -dname "CN=localhost" -ext "SAN=dns:localhost,ip:127.0.0.1" \ + * -keystore localhost.p12 -storetype PKCS12 -storepass changeit -keypass changeit + * + * keytool -genkeypair -alias otherhost -keyalg RSA -keysize 2048 -validity 9000 \ + * -dname "CN=otherhost.invalid" -ext "SAN=dns:otherhost.invalid" \ + * -keystore otherhost.p12 -storetype PKCS12 -storepass changeit -keypass changeit + */ +class OkHttpTrustEverythingTest { + + private static final String KEYSTORE_PASSWORD = "changeit"; + + /** Certificate issued for localhost: valid for the host the tests connect to. */ + private static final String LOCALHOST_KEYSTORE = "/ssl/localhost.p12"; + + /** Certificate issued for another host name: trusted under trust-all, wrong name. */ + private static final String OTHERHOST_KEYSTORE = "/ssl/otherhost.p12"; + + private WireMockServer server; + + @AfterEach + void stopServer() { + if (server != null) { + server.stop(); + server = null; + } + } + + @Test + void trustAllContextUsesTls() { + assertEquals( + "TLS", + HttpProtocol.trustAllSslContext.getProtocol(), + "the trust-all SSLContext should be a TLS context"); + } + + @Test + void selfSignedCertificateRejectedByDefault() throws Exception { + // http.trust.everything defaults to false: an unvalidatable certificate + // must not be accepted + startServer(LOCALHOST_KEYSTORE); + final HttpProtocol protocol = protocol(config()); + assertThrows( + SSLHandshakeException.class, + () -> fetch(protocol, "/default"), + "self-signed certificates must be rejected by default"); + } + + @Test + void trustAllFetchesServerWithSelfSignedCertificate() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + startServer(LOCALHOST_KEYSTORE); + final ProtocolResponse response = fetch(protocol(conf), "/trustall"); + assertEquals(200, response.getStatusCode(), "the self-signed certificate is accepted"); + } + + @Test + void hostnameIsStillVerified() throws Exception { + // the certificate is issued for another host name: trusting any + // certificate must not imply accepting any name + final Config conf = config(); + conf.put("http.trust.everything", true); + startServer(OTHERHOST_KEYSTORE); + assertThrows( + SSLPeerUnverifiedException.class, + () -> fetch(protocol(conf), "/hostname"), + "the hostname verifier should not accept any name unconditionally"); + } + + @Test + void hostnameVerificationCanBeDisabledSeparately() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + conf.put("http.verify.hostnames", false); + startServer(OTHERHOST_KEYSTORE); + final ProtocolResponse response = fetch(protocol(conf), "/nohostnamecheck"); + assertEquals(200, response.getStatusCode(), "the name mismatch is accepted as configured"); + } + + @Test + void basicAuthIsWithheldFromUnauthenticatedServers() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + conf.put("http.basicauth.user", "user"); + conf.put("http.basicauth.password", "secret"); + startServer(LOCALHOST_KEYSTORE); + fetch(protocol(conf), "/basicauth"); + server.verify( + 1, getRequestedFor(urlPathEqualTo("/basicauth")).withoutHeader("Authorization")); + } + + @Test + void basicAuthIsSentWhenExplicitlyAllowed() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + conf.put("http.credentials.allow.insecure", true); + conf.put("http.basicauth.user", "user"); + conf.put("http.basicauth.password", "secret"); + startServer(LOCALHOST_KEYSTORE); + fetch(protocol(conf), "/basicauth"); + final String expected = + "Basic " + + Base64.getEncoder() + .encodeToString("user:secret".getBytes(StandardCharsets.UTF_8)); + server.verify( + 1, + getRequestedFor(urlPathEqualTo("/basicauth")) + .withHeader("Authorization", equalTo(expected))); + } + + @Test + void credentialCustomHeadersAreWithheldFromUnauthenticatedServers() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + conf.put("http.custom.headers", List.of("X-Api-Key=s3cret", "X-Trace=public")); + startServer(LOCALHOST_KEYSTORE); + fetch(protocol(conf), "/customheaders"); + server.verify( + 1, getRequestedFor(urlPathEqualTo("/customheaders")).withoutHeader("X-Api-Key")); + server.verify( + 1, + getRequestedFor(urlPathEqualTo("/customheaders")) + .withHeader("X-Trace", equalTo("public"))); + } + + @Test + void credentialCustomHeadersAreSentWhenExplicitlyAllowed() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + conf.put("http.credentials.allow.insecure", true); + conf.put("http.custom.headers", List.of("X-Api-Key=s3cret", "X-Trace=public")); + startServer(LOCALHOST_KEYSTORE); + fetch(protocol(conf), "/customheaders"); + server.verify( + 1, + getRequestedFor(urlPathEqualTo("/customheaders")) + .withHeader("X-Api-Key", equalTo("s3cret"))); + } + + @Test + void cookiesAreWithheldFromUnauthenticatedServers() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + conf.put("http.use.cookies", true); + startServer(LOCALHOST_KEYSTORE); + fetch(protocol(conf), "/cookies", metadata()); + server.verify(1, getRequestedFor(urlPathEqualTo("/cookies")).withoutHeader("Cookie")); + } + + @Test + void cookiesAreSentWhenExplicitlyAllowed() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + conf.put("http.credentials.allow.insecure", true); + conf.put("http.use.cookies", true); + startServer(LOCALHOST_KEYSTORE); + fetch(protocol(conf), "/cookies", metadata()); + server.verify( + 1, + getRequestedFor(urlPathEqualTo("/cookies")).withHeader("Cookie", equalTo("sid=x"))); + } + + @Test + void basicAuthIsWithheldOverCleartextHttp() throws Exception { + // a cleartext http:// request does not authenticate the server either + final Config conf = config(); + conf.put("http.basicauth.user", "user"); + conf.put("http.basicauth.password", "secret"); + startServer(LOCALHOST_KEYSTORE); + final ProtocolResponse response = + fetchUrl(protocol(conf), httpUrl("/cleartext"), new Metadata()); + assertEquals(200, response.getStatusCode(), "the connection must succeed"); + server.verify( + 1, getRequestedFor(urlPathEqualTo("/cleartext")).withoutHeader("Authorization")); + } + + @Test + void basicAuthIsSentOverCleartextHttpWhenExplicitlyAllowed() throws Exception { + final Config conf = config(); + conf.put("http.credentials.allow.insecure", true); + conf.put("http.basicauth.user", "user"); + conf.put("http.basicauth.password", "secret"); + startServer(LOCALHOST_KEYSTORE); + fetchUrl(protocol(conf), httpUrl("/cleartext"), new Metadata()); + final String expected = "Basic " + base64("user:secret"); + server.verify( + 1, + getRequestedFor(urlPathEqualTo("/cleartext")) + .withHeader("Authorization", equalTo(expected))); + } + + @Test + void credentialHeaderNamesCanBeConfigured() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + conf.put("http.credentials.headers", List.of("X-Auth-Token")); + conf.put("http.custom.headers", List.of("X-Auth-Token=token1", "X-Other=plain")); + startServer(LOCALHOST_KEYSTORE); + fetch(protocol(conf), "/configuredheaders"); + server.verify( + 1, + getRequestedFor(urlPathEqualTo("/configuredheaders")) + .withoutHeader("X-Auth-Token")); + server.verify( + 1, + getRequestedFor(urlPathEqualTo("/configuredheaders")) + .withHeader("X-Other", equalTo("plain"))); + } + + /** http.credentials.headers adds to the built-in names, it does not replace them. */ + @Test + void configuredHeaderNamesDoNotDropTheBuiltInOnes() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + conf.put("http.credentials.headers", List.of("x-auth-token")); + conf.put( + "http.custom.headers", + List.of("Authorization=Basic c2VjcmV0", "X-Api-Key=key1", "Cookie=sid=x")); + startServer(LOCALHOST_KEYSTORE); + fetch(protocol(conf), "/builtinheaders"); + server.verify( + 1, + getRequestedFor(urlPathEqualTo("/builtinheaders")) + .withoutHeader("Authorization") + .withoutHeader("X-Api-Key") + .withoutHeader("Cookie")); + } + + /** + * Credentials must also be withheld when hostname verification is switched off: a valid + * certificate for a different name, accepted only because the name is not checked, does not + * authenticate the server any more than a trust-all chain does. + */ + @Test + void credentialsAreWithheldWhenHostnameVerificationIsDisabled() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + conf.put("http.verify.hostnames", false); + conf.put("http.basicauth.user", "user"); + conf.put("http.basicauth.password", "secret"); + startServer(OTHERHOST_KEYSTORE); + final ProtocolResponse response = fetch(protocol(conf), "/nohostnamecheck"); + assertEquals(200, response.getStatusCode(), "the connection succeeds as configured"); + server.verify( + 1, + getRequestedFor(urlPathEqualTo("/nohostnamecheck")).withoutHeader("Authorization")); + } + + /** The opt-in still sends them on that path. */ + @Test + void credentialsAreSentWhenHostnameVerificationIsDisabledAndInsecureAllowed() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + conf.put("http.verify.hostnames", false); + conf.put("http.credentials.allow.insecure", true); + conf.put("http.basicauth.user", "user"); + conf.put("http.basicauth.password", "secret"); + startServer(OTHERHOST_KEYSTORE); + fetch(protocol(conf), "/nohostnamecheckinsecure"); + final String expected = "Basic " + base64("user:secret"); + server.verify( + 1, + getRequestedFor(urlPathEqualTo("/nohostnamecheckinsecure")) + .withHeader("Authorization", equalTo(expected))); + } + + /** Credential headers set per request through metadata go through the same policy. */ + @Test + void credentialHeaderSetByRequestIsWithheldFromUnauthenticatedServers() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + startServer(LOCALHOST_KEYSTORE); + final Metadata md = new Metadata(); + md.setValue("protocol.set-header", "X-Api-Key=s3cret"); + md.setValue("protocol.set-header", "X-Trace=public"); + fetch(protocol(conf), "/setheaders", md); + server.verify(1, getRequestedFor(urlPathEqualTo("/setheaders")).withoutHeader("X-Api-Key")); + server.verify( + 1, + getRequestedFor(urlPathEqualTo("/setheaders")) + .withHeader("X-Trace", equalTo("public"))); + } + + @Test + void credentialHeaderSetByRequestIsSentWhenExplicitlyAllowed() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + conf.put("http.credentials.allow.insecure", true); + startServer(LOCALHOST_KEYSTORE); + final Metadata md = new Metadata(); + md.setValue("protocol.set-header", "X-Api-Key=s3cret"); + fetch(protocol(conf), "/setheadersallowed", md); + server.verify( + 1, + getRequestedFor(urlPathEqualTo("/setheadersallowed")) + .withHeader("X-Api-Key", equalTo("s3cret"))); + } + + /** + * The decision itself over the full configuration matrix, including the positive case of a + * normally validated HTTPS connection, which the WireMock tests above cannot express because + * their certificates are self-signed. + */ + @Test + void credentialsAllowedDecision() { + // default: chains validated, hostnames checked -> https is authenticated + HttpProtocol p = protocol(config()); + assertTrue(p.credentialsAllowed("https://example.org/"), "validated https sends"); + assertFalse(p.credentialsAllowed("http://example.org/"), "cleartext never sends"); + // trust-all: chains are not validated + final Config trustAll = config(); + trustAll.put("http.trust.everything", true); + assertFalse(protocol(trustAll).credentialsAllowed("https://example.org/")); + // hostname verification disabled: a name mismatch is not caught + final Config noHostCheck = config(); + noHostCheck.put("http.verify.hostnames", false); + assertFalse(protocol(noHostCheck).credentialsAllowed("https://example.org/")); + // the opt-in covers every case + final Config allowInsecure = config(); + allowInsecure.put("http.credentials.allow.insecure", true); + assertTrue(protocol(allowInsecure).credentialsAllowed("http://example.org/")); + } + + /** + * An automatic redirect from trusted HTTPS to cleartext HTTP must not forward credentials: the + * policy is enforced on every hop, not just the initial URL. The self-signed test certificate + * is trusted via the client transport (without flipping http.trust.everything, which would also + * flip the policy), so the initial HTTPS hop is authenticated and sends credentials while the + * HTTP hop must not receive them. + */ + @Test + void credentialsAreStrippedOnHttpsToHttpRedirect() throws Exception { + final Config conf = config(); + conf.put("http.allow.redirects", true); + conf.put("http.basicauth.user", "user"); + conf.put("http.basicauth.password", "secret"); + conf.put("http.custom.headers", List.of("X-Api-Key=s3cret")); + startServer(LOCALHOST_KEYSTORE); + final HttpProtocol protocol = protocol(conf); + trustTestKeystore(protocol, LOCALHOST_KEYSTORE); + server.stubFor( + get(urlPathEqualTo("/redirect")) + .atPriority(1) + .willReturn( + aResponse() + .withStatus(302) + .withHeader("Location", httpUrl("/target")))); + final ProtocolResponse response = + fetchUrl( + protocol, + "https://localhost:" + server.httpsPort() + "/redirect", + new Metadata()); + assertEquals(200, response.getStatusCode(), "the redirect is followed"); + final String expected = "Basic " + base64("user:secret"); + server.verify( + 1, + getRequestedFor(urlPathEqualTo("/redirect")) + .withHeader("Authorization", equalTo(expected))); + server.verify(1, getRequestedFor(urlPathEqualTo("/target")).withoutHeader("Authorization")); + server.verify(1, getRequestedFor(urlPathEqualTo("/target")).withoutHeader("X-Api-Key")); + } + + /** + * With the opt-in, the same redirect forwards custom credential headers to the HTTP hop. + * (OkHttp itself strips Authorization on a scheme downgrade even then, which is its safe + * default; the policy opt-in covers the headers StormCrawler controls.) + */ + @Test + void credentialsAreForwardedOnHttpsToHttpRedirectWhenInsecureAllowed() throws Exception { + final Config conf = config(); + conf.put("http.allow.redirects", true); + conf.put("http.credentials.allow.insecure", true); + conf.put("http.basicauth.user", "user"); + conf.put("http.basicauth.password", "secret"); + conf.put("http.custom.headers", List.of("X-Api-Key=s3cret")); + startServer(LOCALHOST_KEYSTORE); + final HttpProtocol protocol = protocol(conf); + trustTestKeystore(protocol, LOCALHOST_KEYSTORE); + server.stubFor( + get(urlPathEqualTo("/redirect-allowed")) + .atPriority(1) + .willReturn( + aResponse() + .withStatus(302) + .withHeader("Location", httpUrl("/target-allowed")))); + fetchUrl( + protocol, + "https://localhost:" + server.httpsPort() + "/redirect-allowed", + new Metadata()); + server.verify( + 1, + getRequestedFor(urlPathEqualTo("/target-allowed")) + .withHeader("X-Api-Key", equalTo("s3cret"))); + } + + /** + * With http.store.headers, the request headers recorded for a hop are the ones sent on it: a + * credential stripped from the cleartext hop must not show up in the stored request either. + */ + @Test + void storedRequestHeadersMatchTheStrippedHop() throws Exception { + final Config conf = config(); + conf.put("http.allow.redirects", true); + conf.put("http.store.headers", true); + conf.put("http.custom.headers", List.of("X-Api-Key=s3cret")); + startServer(LOCALHOST_KEYSTORE); + final HttpProtocol protocol = protocol(conf); + trustTestKeystore(protocol, LOCALHOST_KEYSTORE); + server.stubFor( + get(urlPathEqualTo("/redirect-stored")) + .atPriority(1) + .willReturn( + aResponse() + .withStatus(302) + .withHeader("Location", httpUrl("/target-stored")))); + final ProtocolResponse response = + fetchUrl( + protocol, + "https://localhost:" + server.httpsPort() + "/redirect-stored", + new Metadata()); + final String stored = + response.getMetadata().getFirstValue(ProtocolResponse.REQUEST_HEADERS_KEY); + assertTrue(stored.contains("/target-stored "), "the last hop is recorded: " + stored); + assertFalse( + stored.toLowerCase(Locale.ROOT).contains("x-api-key"), + "the stripped header must not be recorded: " + stored); + } + + /** + * A Proxy-Authorization header configured in http.custom.headers is withheld like any other + * credential. On a direct connection it would reach the crawled server, so a redirect to + * cleartext HTTP must strip it too. + */ + @Test + void staticProxyAuthorizationIsStrippedOnHttpsToHttpRedirect() throws Exception { + final Config conf = config(); + conf.put("http.allow.redirects", true); + conf.put("http.custom.headers", List.of("Proxy-Authorization=Basic cHJveHk6c2VjcmV0")); + startServer(LOCALHOST_KEYSTORE); + final HttpProtocol protocol = protocol(conf); + trustTestKeystore(protocol, LOCALHOST_KEYSTORE); + server.stubFor( + get(urlPathEqualTo("/redirect-proxyauth")) + .atPriority(1) + .willReturn( + aResponse() + .withStatus(302) + .withHeader("Location", httpUrl("/target-proxyauth")))); + fetchUrl( + protocol, + "https://localhost:" + server.httpsPort() + "/redirect-proxyauth", + new Metadata()); + server.verify( + 1, + getRequestedFor(urlPathEqualTo("/redirect-proxyauth")) + .withHeader("Proxy-Authorization", equalTo("Basic cHJveHk6c2VjcmV0"))); + server.verify( + 1, + getRequestedFor(urlPathEqualTo("/target-proxyauth")) + .withoutHeader("Proxy-Authorization")); + } + + /** + * Trusts the self-signed test keystore on the protocol transport without changing the + * credential policy: http.trust.everything stays false, so credentialsAllowed still treats the + * HTTPS URL as authenticated, while the TLS handshake succeeds. + */ + private void trustTestKeystore(HttpProtocol protocol, String keystoreResource) + throws Exception { + final KeyStore trustStore = KeyStore.getInstance("PKCS12"); + try (InputStream in = getClass().getResourceAsStream(keystoreResource)) { + trustStore.load(in, KEYSTORE_PASSWORD.toCharArray()); + } + final TrustManagerFactory tmf = + TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + tmf.init(trustStore); + final X509TrustManager trustManager = (X509TrustManager) tmf.getTrustManagers()[0]; + final SSLContext sslContext = SSLContext.getInstance("TLS"); + sslContext.init(null, new TrustManager[] {trustManager}, new SecureRandom()); + final Field clientField = HttpProtocol.class.getDeclaredField("client"); + clientField.setAccessible(true); + final OkHttpClient client = (OkHttpClient) clientField.get(protocol); + final OkHttpClient trusted = + client.newBuilder() + .sslSocketFactory(sslContext.getSocketFactory(), trustManager) + .build(); + clientField.set(protocol, trusted); + } + + /** Metadata as an outlink would inherit it, with a cookie scoped to the server. */ + private Metadata metadata() { + final Metadata md = new Metadata(); + md.setValue("protocol.set-cookie", "sid=x; Path=/"); + md.setValue("protocol.set-cookie-origin", "https://localhost:" + server.httpsPort() + "/"); + return md; + } + + private String base64(String value) { + return Base64.getEncoder().encodeToString(value.getBytes(StandardCharsets.UTF_8)); + } + + private String httpUrl(String path) { + return "http://localhost:" + server.port() + path; + } + + private Config config() { + final Config conf = new Config(); + conf.put("http.agent.name", "test"); + conf.put("http.agent.version", "1.0"); + conf.put("http.agent.description", "test"); + conf.put("http.agent.url", "http://test.example.com"); + conf.put("http.agent.email", "test@example.com"); + conf.put("protocol.md.prefix", "protocol."); + return conf; + } + + private HttpProtocol protocol(Config conf) { + final HttpProtocol protocol = new HttpProtocol(); + protocol.configure(conf); + return protocol; + } + + private ProtocolResponse fetch(HttpProtocol protocol, String path) throws Exception { + return fetch(protocol, path, new Metadata()); + } + + private ProtocolResponse fetch(HttpProtocol protocol, String path, Metadata md) + throws Exception { + return fetchUrl(protocol, "https://localhost:" + server.httpsPort() + path, md); + } + + private ProtocolResponse fetchUrl(HttpProtocol protocol, String url, Metadata md) + throws Exception { + return protocol.getProtocolOutput(url, md); + } + + /** + * Starts an HTTPS server on a random port presenting the certificate of the keystore. The + * keystore is copied to a temporary file as WireMock reads it from the file system. + */ + private void startServer(String keystoreResource) throws Exception { + final Path keystoreFile = Files.createTempFile("wiremock-keystore", ".p12"); + keystoreFile.toFile().deleteOnExit(); + try (InputStream in = + OkHttpTrustEverythingTest.class.getResourceAsStream(keystoreResource)) { + Files.copy(in, keystoreFile, StandardCopyOption.REPLACE_EXISTING); + } + + server = + new WireMockServer( + WireMockConfiguration.options() + .dynamicPort() + .dynamicHttpsPort() + .keystorePath(keystoreFile.toAbsolutePath().toString()) + .keystorePassword(KEYSTORE_PASSWORD) + .keyManagerPassword(KEYSTORE_PASSWORD) + .keystoreType("PKCS12") + .notifier(new ConsoleNotifier(false))); + server.start(); + server.stubFor(any(anyUrl()).willReturn(ok("Success!"))); + } +} diff --git a/core/src/test/resources/ssl/localhost.p12 b/core/src/test/resources/ssl/localhost.p12 new file mode 100644 index 000000000..222897c69 Binary files /dev/null and b/core/src/test/resources/ssl/localhost.p12 differ diff --git a/core/src/test/resources/ssl/otherhost.p12 b/core/src/test/resources/ssl/otherhost.p12 new file mode 100644 index 000000000..d9543037e Binary files /dev/null and b/core/src/test/resources/ssl/otherhost.p12 differ diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc index c57fca469..781fb5670 100644 --- a/docs/src/main/asciidoc/configuration.adoc +++ b/docs/src/main/asciidoc/configuration.adoc @@ -261,7 +261,10 @@ header. | http.content.partial.as.trimmed | false | Accepts partially fetched content in OKHTTP. | http.filter.ipaddress.include | - | (OkHttp only) Comma-separated list (or YAML list) of allowed IP ranges. If empty, all addresses are allowed unless excluded. See <>. | http.filter.ipaddress.exclude | - | (OkHttp only) Comma-separated list (or YAML list) of blocked IP ranges. See <>. -| http.trust.everything | true | If true, trust all SSL/TLS connections. +| http.trust.everything | false | (OkHttp only) If true, accept any TLS certificate, including self-signed, expired or otherwise invalid ones. The servers are then not authenticated: anyone able to answer for the host name receives everything sent to them. Credentials (basic auth, credential headers, cookies) are withheld from cleartext http:// requests and from https:// requests on such connections unless `http.credentials.allow.insecure` is set to true. +| http.verify.hostnames | true | (OkHttp only) If true, check that the certificate presented by the server matches the host name contacted. Independent of `http.trust.everything`: a valid certificate for a different name is accepted when this is false. Credentials are then withheld unless `http.credentials.allow.insecure` is set to true. +| http.credentials.allow.insecure | false | (OkHttp only) If true, send credentials (basic auth, credential headers, cookies) even on connections which do not authenticate the server: cleartext http:// urls and, when `http.trust.everything` is true or `http.verify.hostnames` is false, https:// urls as well. +| http.credentials.headers | authorization, proxy-authorization, cookie, x-api-key | (OkHttp only) Header names (case-insensitive) which carry credentials and are therefore withheld from requests to servers which were not authenticated. Applies to the headers configured with `http.custom.headers` and to headers set by request. Names given here are added to the four defaults, which are always treated as credentials. | topology.message.timeout.secs | -1 | OKHTTP message timeout. |=== diff --git a/docs/src/main/asciidoc/extending.adoc b/docs/src/main/asciidoc/extending.adoc index 0837db0a7..25f65ab7e 100644 --- a/docs/src/main/asciidoc/extending.adoc +++ b/docs/src/main/asciidoc/extending.adoc @@ -392,7 +392,7 @@ Keep the pool size (idle + active) under 1000. For large crawls, increase `proto ==== SSL/TLS -By default, StormCrawler's OkHttp protocol trusts all certificates (`http.trust.everything: true`). For production crawls over untrusted networks, consider setting this to `false` and configuring a proper trust store. +By default, StormCrawler's OkHttp protocol validates the certificate chains of HTTPS servers and checks that the certificate matches the host name contacted (`http.trust.everything: false`, `http.verify.hostnames: true`). For crawls of hosts whose certificates cannot be validated, e.g. self-signed ones, set `http.trust.everything` to `true`: the servers are then not authenticated, so credentials (basic auth, credential headers in `http.custom.headers`, cookies) are withheld unless `http.credentials.allow.insecure` is set to `true`. Credentials are also withheld from cleartext http:// requests, which never authenticate the server. Which header names count as credentials is configured with `http.credentials.headers`. Hostname checking is a separate decision and can be disabled with `http.verify.hostnames: false`; the servers are then not authenticated either, and credentials are withheld in the same way. ==== Authentication @@ -404,7 +404,7 @@ http.basicauth.user: "username" http.basicauth.password: "password" ---- -These credentials are sent as an `Authorization` header with every request. For per-site authentication, use metadata-driven headers instead (see <> in the Internals section). +These credentials are sent as an `Authorization` header with requests to servers authenticated by their TLS certificate; they are withheld from cleartext http:// requests and from https:// requests when `http.trust.everything` is enabled or `http.verify.hostnames` is disabled, unless `http.credentials.allow.insecure` is set to `true`. For per-site authentication, use metadata-driven headers instead (see <> in the Internals section). ==== Proxy Authentication diff --git a/external/opensearch/archetype/src/main/resources/archetype-resources/crawler-conf.yaml b/external/opensearch/archetype/src/main/resources/archetype-resources/crawler-conf.yaml index f62103faf..f2d9860f5 100644 --- a/external/opensearch/archetype/src/main/resources/archetype-resources/crawler-conf.yaml +++ b/external/opensearch/archetype/src/main/resources/archetype-resources/crawler-conf.yaml @@ -72,6 +72,12 @@ config: http.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol" https.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol" + # Accept any TLS certificate, e.g. self-signed ones? The servers are not + # authenticated then; credentials (basic auth, credential headers, cookies) + # are withheld over such connections unless + # http.credentials.allow.insecure is set to true. + http.trust.everything: false + # The maximum number of bytes for returned HTTP response bodies. # The fetched page will be trimmed to 65KB in this case # Set -1 to disable the limit. diff --git a/external/solr/archetype/src/main/resources/archetype-resources/crawler-conf.yaml b/external/solr/archetype/src/main/resources/archetype-resources/crawler-conf.yaml index f62103faf..f2d9860f5 100644 --- a/external/solr/archetype/src/main/resources/archetype-resources/crawler-conf.yaml +++ b/external/solr/archetype/src/main/resources/archetype-resources/crawler-conf.yaml @@ -72,6 +72,12 @@ config: http.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol" https.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol" + # Accept any TLS certificate, e.g. self-signed ones? The servers are not + # authenticated then; credentials (basic auth, credential headers, cookies) + # are withheld over such connections unless + # http.credentials.allow.insecure is set to true. + http.trust.everything: false + # The maximum number of bytes for returned HTTP response bodies. # The fetched page will be trimmed to 65KB in this case # Set -1 to disable the limit.