From f9ce12bb51b0aa26f75479ae5b083abe1b4b64de Mon Sep 17 00:00:00 2001 From: abhinav-phi Date: Sun, 6 Sep 2026 04:45:19 +0530 Subject: [PATCH 1/7] OKHttp: authenticated TLS by default, credentials withheld from unauthenticated servers (#2079) http.trust.everything shipped enabled with a static SSLContext('SSL'), a no-op trust manager and a hostname verifier that accepted any name: anybody able to answer for a host name received the Authorization header built from http.basicauth.*, credential headers from http.custom.headers and the replayed cookies. - http.trust.everything now defaults to false: certificate chains are validated and the servers are authenticated, as a normal TLS client would do. Self-signed intranet hosts need the operator to opt in - the trust-all context is built from 'TLS' instead of 'SSL' - hostname verification is a separate decision (http.verify.hostnames, default true) and is no longer disabled as a side effect of trusting any certificate - credentials are withheld over connections whose server certificate was not validated, unless http.credentials.allow.insecure is set: basic auth, credential headers (authorization, proxy-authorization, cookie, x-api-key) in http.custom.headers, headers set by request and cookies are all covered - every insecure state logs a WARN naming the keys, and the keys are surfaced in crawler-default.yaml and the archetype crawler-conf.yaml files --- .../archetype-resources/crawler-conf.yaml | 6 + .../protocol/okhttp/HttpProtocol.java | 130 ++++++-- core/src/main/resources/crawler-default.yaml | 24 ++ .../okhttp/OkHttpTrustEverythingTest.java | 289 ++++++++++++++++++ core/src/test/resources/ssl/localhost.p12 | Bin 0 -> 2606 bytes core/src/test/resources/ssl/otherhost.p12 | Bin 0 -> 2622 bytes docs/src/main/asciidoc/configuration.adoc | 4 +- docs/src/main/asciidoc/extending.adoc | 4 +- .../archetype-resources/crawler-conf.yaml | 6 + .../archetype-resources/crawler-conf.yaml | 6 + 10 files changed, 447 insertions(+), 22 deletions(-) create mode 100644 core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java create mode 100644 core/src/test/resources/ssl/localhost.p12 create mode 100644 core/src/test/resources/ssl/otherhost.p12 diff --git a/archetype/src/main/resources/archetype-resources/crawler-conf.yaml b/archetype/src/main/resources/archetype-resources/crawler-conf.yaml index bc57e3714..6db628282 100644 --- a/archetype/src/main/resources/archetype-resources/crawler-conf.yaml +++ b/archetype/src/main/resources/archetype-resources/crawler-conf.yaml @@ -72,6 +72,12 @@ config: http.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol" https.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol" + # Accept any TLS certificate, e.g. self-signed ones? The servers are not + # authenticated then; credentials (basic auth, credential headers, cookies) + # are withheld over such connections unless + # http.credentials.allow.insecure is set to true. + http.trust.everything: false + # The maximum number of bytes for returned HTTP response bodies. # The fetched page will be trimmed to 65KB in this case # Set -1 to disable the limit. diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java index 927508b43..71183a58a 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java @@ -34,12 +34,11 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import javax.net.ssl.HostnameVerifier; import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLSession; import javax.net.ssl.SSLSocketFactory; import javax.net.ssl.TrustManager; import javax.net.ssl.X509TrustManager; @@ -108,6 +107,23 @@ public class HttpProtocol extends AbstractHttpProtocol { // makes sure that a missing cookie origin is reported once and not for every url private final AtomicBoolean missingCookieOriginLogged = new AtomicBoolean(); + // makes sure that withheld cookies are reported once and not for every url + private final AtomicBoolean withheldCookiesLogged = new AtomicBoolean(); + + // whether credentials (basic auth, credential headers, cookies) may be sent: + // false when the servers are not authenticated and + // http.credentials.allow.insecure is not enabled + private boolean sendCredentials = true; + + /** Header names carrying credentials, see {@link #isCredentialHeader(String)}. */ + private static final Set CREDENTIAL_HEADERS = + Set.of( + HttpHeaders.AUTHORIZATION.toLowerCase(Locale.ROOT), + HttpHeaders.PROXY_AUTHORIZATION.toLowerCase(Locale.ROOT), + // the cookie header is not a constant in HttpHeaders + "cookie", + "x-api-key"); + private OkHttpClient.Builder builder; private static final TrustManager[] trustAllCerts = @@ -130,11 +146,12 @@ public java.security.cert.X509Certificate[] getAcceptedIssuers() { } }; - private static final SSLContext trustAllSslContext; + // package-private so that the OkHttpTrustEverythingTest can check the protocol + static final SSLContext trustAllSslContext; static { try { - trustAllSslContext = SSLContext.getInstance("SSL"); + trustAllSslContext = SSLContext.getInstance("TLS"); trustAllSslContext.init(null, trustAllCerts, new java.security.SecureRandom()); } catch (Exception e) { throw new RuntimeException(e); @@ -158,6 +175,37 @@ public void configure(Config conf) { this.partialContentAsTrimmed = ConfUtils.getBoolean(conf, "http.content.partial.as.trimmed", false); + /* + * certificate trust and hostname verification are separate decisions: + * accepting any certificate does not imply accepting any name + */ + final boolean trustEverything = ConfUtils.getBoolean(conf, "http.trust.everything", false); + final boolean verifyHostnames = ConfUtils.getBoolean(conf, "http.verify.hostnames", true); + // credentials are withheld over connections whose server certificate + // was not validated, unless explicitly opted in + final boolean insecureCredentialsAllowed = + ConfUtils.getBoolean(conf, "http.credentials.allow.insecure", false); + this.sendCredentials = !trustEverything || insecureCredentialsAllowed; + + if (trustEverything) { + LOG.warn( + "http.trust.everything is enabled: TLS certificate chains are accepted without " + + "validation, the identity of the servers is not authenticated. Anybody " + + "able to answer for the host name receives everything sent to them."); + } + if (!verifyHostnames) { + LOG.warn( + "http.verify.hostnames is disabled: the certificates are not checked against " + + "the host name either, the identity of the servers is not authenticated."); + } + if (trustEverything && !sendCredentials) { + LOG.warn( + "Credentials configured with http.basicauth.*, credential headers in " + + "http.custom.headers and cookies are withheld because the servers " + + "are not authenticated. Set http.credentials.allow.insecure to true " + + "to send them anyway."); + } + builder = new OkHttpClient.Builder() .retryOnConnectionFailure( @@ -223,15 +271,33 @@ public void configure(Config conf) { // use a basic auth? if (StringUtils.isNotBlank(basicAuthUser)) { final String basicAuthPass = ConfUtils.getString(conf, "http.basicauth.password", ""); - final String encoding = - Base64.getEncoder() - .encodeToString( - (basicAuthUser + ":" + basicAuthPass) - .getBytes(StandardCharsets.UTF_8)); - customRequestHeaders.add(new KeyValue(HttpHeaders.AUTHORIZATION, "Basic " + encoding)); + if (sendCredentials) { + final String encoding = + Base64.getEncoder() + .encodeToString( + (basicAuthUser + ":" + basicAuthPass) + .getBytes(StandardCharsets.UTF_8)); + customRequestHeaders.add( + new KeyValue(HttpHeaders.AUTHORIZATION, "Basic " + encoding)); + } else { + LOG.warn( + "Basic authentication configured with http.basicauth.user is withheld " + + "because the servers are not authenticated (http.trust.everything). " + + "Set http.credentials.allow.insecure to true to send it anyway."); + } } - customHeaders.forEach(customRequestHeaders::add); + for (KeyValue customHeader : customHeaders) { + if (!sendCredentials && isCredentialHeader(customHeader.getKey())) { + LOG.warn( + "Custom header {} is withheld because the servers are not authenticated " + + "(http.trust.everything). Set http.credentials.allow.insecure to " + + "true to send it anyway.", + customHeader.getKey()); + continue; + } + customRequestHeaders.add(customHeader); + } // optionally block connections to forbidden IP address ranges // (e.g. localhost/loopback, private/site-local addresses), see @@ -245,15 +311,11 @@ public void configure(Config conf) { builder.addNetworkInterceptor(new HTTPHeadersInterceptor()); } - if (ConfUtils.getBoolean(conf, "http.trust.everything", true)) { + if (trustEverything) { builder.sslSocketFactory(trustAllSslSocketFactory, (X509TrustManager) trustAllCerts[0]); - builder.hostnameVerifier( - new HostnameVerifier() { - @Override - public boolean verify(String hostname, SSLSession session) { - return true; - } - }); + } + if (!verifyHostnames) { + builder.hostnameVerifier((hostname, session) -> true); } builder.eventListenerFactory( @@ -285,6 +347,15 @@ public EventListener create(Call call) { } private void addCookiesToRequest(Builder rb, String url, Metadata md) { + if (!sendCredentials) { + if (withheldCookiesLogged.compareAndSet(false, true)) { + LOG.warn( + "Cookies are withheld because the servers are not authenticated " + + "(http.trust.everything). Set http.credentials.allow.insecure to " + + "true to send them anyway."); + } + return; + } final String[] cookieStrings = md.getValues(RESPONSE_COOKIES_HEADER, protocolMetadataPrefix); if (cookieStrings == null || cookieStrings.length == 0) { @@ -337,12 +408,33 @@ private URL getCookieOrigin(Metadata md, String url) { } } + /** + * Returns true when the header carries credentials which must not be disclosed to servers that + * were not authenticated. Covers the standard credential headers plus any header commonly used + * for API keys; cookie headers are handled separately in {@link #addCookiesToRequest}. + */ + private static boolean isCredentialHeader(String name) { + if (name == null) { + return false; + } + final String normalised = name.trim().toLowerCase(Locale.ROOT); + return CREDENTIAL_HEADERS.contains(normalised); + } + protected void addHeadersToRequest(Builder rb, Metadata md) { final String[] headerStrings = md.getValues(SET_HEADER_BY_REQUEST, protocolMetadataPrefix); if (headerStrings != null && headerStrings.length > 0) { for (String hs : headerStrings) { KeyValue h = KeyValue.build(hs); + if (!sendCredentials && isCredentialHeader(h.getKey())) { + LOG.warn( + "Header {} set by request is withheld because the servers are not " + + "authenticated (http.trust.everything). Set " + + "http.credentials.allow.insecure to true to send it anyway.", + h.getKey()); + continue; + } rb.addHeader(h.getKey(), h.getValue()); } } diff --git a/core/src/main/resources/crawler-default.yaml b/core/src/main/resources/crawler-default.yaml index 2e251dd5e..273e90464 100644 --- a/core/src/main/resources/crawler-default.yaml +++ b/core/src/main/resources/crawler-default.yaml @@ -149,6 +149,30 @@ config: # Follow redirect HTTP responses: http.allow.redirects: false + # Accept any TLS certificate, including self-signed, expired or otherwise + # invalid ones (okhttp protocol only)? The certificate chains are accepted + # without validation, i.e. the servers are not authenticated: anyone able to + # answer for the host name receives everything sent to them. Needed for + # crawling hosts with unvalidatable certificates, e.g. self-signed ones, but + # also disables protection against man-in-the-middle attacks. Credentials + # (basic auth, credential headers, cookies) are withheld over such + # connections unless http.credentials.allow.insecure is set to true. + http.trust.everything: false + + # Check that the certificate presented by the server matches the host name + # contacted (okhttp protocol only)? Disabling this is independent of + # http.trust.everything: a valid certificate for a different name is then + # accepted. Hostname verification does not need to be disabled for hosts + # with self-signed certificates when http.trust.everything is enabled. + http.verify.hostnames: true + + # Send credentials (basic auth, credential headers, cookies) even over + # connections whose server certificate was not validated, i.e. when + # http.trust.everything is true (okhttp protocol only)? Without it, + # credentials are withheld so that they are not disclosed to servers which + # were never authenticated. + http.credentials.allow.insecure: false + # IP address filtering (okhttp protocol only). Optionally limit or block # connections to IP address ranges once the host name has been resolved. This # prevents information leakage to a public index when a DNS entry points to a diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java new file mode 100644 index 000000000..82d1c136c --- /dev/null +++ b/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java @@ -0,0 +1,289 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.stormcrawler.protocol.okhttp; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; +import com.sun.net.httpserver.HttpsConfigurator; +import com.sun.net.httpserver.HttpsServer; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.security.KeyStore; +import java.util.Base64; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import org.apache.storm.Config; +import org.apache.stormcrawler.Metadata; +import org.apache.stormcrawler.protocol.ProtocolResponse; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +/** + * Regression tests for https://github.com/apache/stormcrawler/issues/2079: the trust-all + * configuration of the okhttp protocol must be an explicit choice, must not disable hostname + * verification and must not disclose credentials to servers which are not authenticated. + */ +class OkHttpTrustEverythingTest { + + private static final char[] KEYSTORE_PASSWORD = "changeit".toCharArray(); + + /** Certificate issued for localhost: valid for the host the tests connect to. */ + private static final String LOCALHOST_KEYSTORE = "/ssl/localhost.p12"; + + /** Certificate issued for another host name: trusted under trust-all, wrong name. */ + private static final String OTHERHOST_KEYSTORE = "/ssl/otherhost.p12"; + + private HttpsServer server; + + private RecordingHandler handler; + + @AfterEach + void stopServer() { + if (server != null) { + server.stop(0); + server = null; + handler = null; + } + } + + @Test + void trustAllContextUsesTls() { + assertEquals( + "TLS", + HttpProtocol.trustAllSslContext.getProtocol(), + "the trust-all SSLContext should be a TLS context"); + } + + @Test + void selfSignedCertificateRejectedByDefault() throws Exception { + // http.trust.everything defaults to false: an unvalidatable certificate + // must not be accepted. Depending on the platform the handshake fails or + // the connection is dropped while it is retried. + startServer(LOCALHOST_KEYSTORE); + final HttpProtocol protocol = protocol(config()); + assertThrows( + Exception.class, + () -> fetch(protocol, "/default"), + "self-signed certificates must be rejected by default"); + } + + @Test + void trustAllFetchesServerWithSelfSignedCertificate() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + startServer(LOCALHOST_KEYSTORE); + final ProtocolResponse response = fetch(protocol(conf), "/trustall"); + assertEquals(200, response.getStatusCode(), "the self-signed certificate is accepted"); + } + + @Test + void hostnameIsStillVerified() throws Exception { + // the certificate is issued for another host name: trusting any + // certificate must not imply accepting any name. Depending on the + // platform the failed verification surfaces as an SSL exception or the + // connection is dropped while it is retried. + final Config conf = config(); + conf.put("http.trust.everything", true); + startServer(OTHERHOST_KEYSTORE); + assertThrows( + Exception.class, + () -> fetch(protocol(conf), "/hostname"), + "the hostname verifier should not accept any name unconditionally"); + } + + @Test + void hostnameVerificationCanBeDisabledSeparately() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + conf.put("http.verify.hostnames", false); + startServer(OTHERHOST_KEYSTORE); + final ProtocolResponse response = fetch(protocol(conf), "/nohostnamecheck"); + assertEquals(200, response.getStatusCode(), "the name mismatch is accepted as configured"); + } + + @Test + void basicAuthIsWithheldFromUnauthenticatedServers() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + conf.put("http.basicauth.user", "user"); + conf.put("http.basicauth.password", "secret"); + startServer(LOCALHOST_KEYSTORE); + final ProtocolResponse response = fetch(protocol(conf), "/basicauth"); + assertEquals(200, response.getStatusCode(), "the connection must succeed"); + assertNull( + handler.lastHeaders.get("authorization"), + "credentials must not be sent to unauthenticated servers"); + } + + @Test + void basicAuthIsSentWhenExplicitlyAllowed() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + conf.put("http.credentials.allow.insecure", true); + conf.put("http.basicauth.user", "user"); + conf.put("http.basicauth.password", "secret"); + startServer(LOCALHOST_KEYSTORE); + fetch(protocol(conf), "/basicauth"); + final String expected = + "Basic " + + Base64.getEncoder() + .encodeToString("user:secret".getBytes(StandardCharsets.UTF_8)); + assertEquals( + expected, + handler.lastHeaders.get("authorization"), + "the opt-in sends credentials"); + } + + @Test + void credentialCustomHeadersAreWithheldFromUnauthenticatedServers() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + conf.put("http.custom.headers", List.of("X-Api-Key=s3cret", "X-Trace=public")); + startServer(LOCALHOST_KEYSTORE); + final ProtocolResponse response = fetch(protocol(conf), "/customheaders"); + assertEquals(200, response.getStatusCode(), "the connection must succeed"); + assertNull( + handler.lastHeaders.get("x-api-key"), "credential headers must be withheld"); + assertEquals("public", handler.lastHeaders.get("x-trace"), "other headers are sent"); + } + + @Test + void credentialCustomHeadersAreSentWhenExplicitlyAllowed() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + conf.put("http.credentials.allow.insecure", true); + conf.put("http.custom.headers", List.of("X-Api-Key=s3cret", "X-Trace=public")); + startServer(LOCALHOST_KEYSTORE); + fetch(protocol(conf), "/customheaders"); + assertEquals("s3cret", handler.lastHeaders.get("x-api-key"), "the opt-in sends credentials"); + assertEquals("public", handler.lastHeaders.get("x-trace")); + } + + @Test + void cookiesAreWithheldFromUnauthenticatedServers() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + conf.put("http.use.cookies", true); + startServer(LOCALHOST_KEYSTORE); + final ProtocolResponse response = fetch(protocol(conf), "/cookies", metadata()); + assertEquals(200, response.getStatusCode(), "the connection must succeed"); + assertNull( + handler.lastHeaders.get("cookie"), + "cookies must not be sent to unauthenticated servers"); + } + + @Test + void cookiesAreSentWhenExplicitlyAllowed() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + conf.put("http.credentials.allow.insecure", true); + conf.put("http.use.cookies", true); + startServer(LOCALHOST_KEYSTORE); + fetch(protocol(conf), "/cookies", metadata()); + assertEquals("sid=x", handler.lastHeaders.get("cookie"), "the opt-in sends cookies"); + } + + /** Metadata as an outlink would inherit it, with a cookie scoped to the server. */ + private Metadata metadata() { + final Metadata md = new Metadata(); + md.setValue("protocol.set-cookie", "sid=x; Path=/"); + md.setValue( + "protocol.set-cookie-origin", + "https://localhost:" + server.getAddress().getPort() + "/"); + return md; + } + + private Config config() { + final Config conf = new Config(); + conf.put("http.agent.name", "test"); + conf.put("http.agent.version", "1.0"); + conf.put("http.agent.description", "test"); + conf.put("http.agent.url", "http://test.example.com"); + conf.put("http.agent.email", "test@example.com"); + conf.put("protocol.md.prefix", "protocol."); + return conf; + } + + private HttpProtocol protocol(Config conf) { + final HttpProtocol protocol = new HttpProtocol(); + protocol.configure(conf); + return protocol; + } + + private ProtocolResponse fetch(HttpProtocol protocol, String path) throws Exception { + return fetch(protocol, path, new Metadata()); + } + + private ProtocolResponse fetch(HttpProtocol protocol, String path, Metadata md) + throws Exception { + return protocol.getProtocolOutput( + "https://localhost:" + server.getAddress().getPort() + path, md); + } + + /** Starts an HTTPS server on a random port presenting the certificate of the keystore. */ + private void startServer(String keystoreResource) throws Exception { + final KeyStore keyStore = KeyStore.getInstance("PKCS12"); + try (InputStream in = OkHttpTrustEverythingTest.class.getResourceAsStream(keystoreResource)) { + keyStore.load(in, KEYSTORE_PASSWORD); + } + final KeyManagerFactory keyManagerFactory = + KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + keyManagerFactory.init(keyStore, KEYSTORE_PASSWORD); + final SSLContext sslContext = SSLContext.getInstance("TLS"); + sslContext.init(keyManagerFactory.getKeyManagers(), null, null); + + server = HttpsServer.create(new InetSocketAddress(java.net.InetAddress.getByName("127.0.0.1"), 0), 0); + server.setHttpsConfigurator(new HttpsConfigurator(sslContext)); + handler = new RecordingHandler(); + server.createContext("/", handler); + server.start(); + } + + /** Records the request headers of the last request received. */ + static class RecordingHandler implements HttpHandler { + + final Map lastHeaders = new ConcurrentHashMap<>(); + + @Override + public void handle(HttpExchange exchange) throws IOException { + for (Map.Entry> header : + exchange.getRequestHeaders().entrySet()) { + lastHeaders.put( + header.getKey().toLowerCase(Locale.ROOT), + String.join(",", header.getValue())); + } + final byte[] body = "Success!".getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().set("Content-Type", "text/html"); + exchange.sendResponseHeaders(200, body.length); + try (OutputStream out = exchange.getResponseBody()) { + out.write(body); + } + } + } +} diff --git a/core/src/test/resources/ssl/localhost.p12 b/core/src/test/resources/ssl/localhost.p12 new file mode 100644 index 0000000000000000000000000000000000000000..222897c6900140ce7e01bf4005095cedd48611ae GIT binary patch literal 2606 zcma)8S5%V?5=}?~gx*1E0i=kOj~?--Mi7y%BB6?av`|GvKp+tVgepbpB25&LCZP0= zgx)(+LXlntCE~JY_xSHV?LN%8GxyHS+uSn%jz*jU3;}TPVM@3-UI%|f52gZV;oxmh z9K7j_t^;tekN+jX@}M}_>oYpzEK(?G|2;)V1E$Er!PL*lzkvIHCSddc8bJPC(gByC zY=Sa-7jtZdd1jK1McYUYpwFj)aTFAyOcY=QKu1aauZ7 zaIFo$!zohuWUwN4un;B(#Xl)V{O z`}>B3>i*c)Z{9b2H@I#)P@P6KEt&6-;NG=Yn_HK3A-8Y?&zWrS*2^&uhB#VIjDuS^ z7+Q3cjXdQp9<(kV#Z6nRXw=)+quiS7ctm1vMNd1H6<)d6rWRt@E8bLB7(P}A;hP-%pFiv|A8U<{x6kY+y(JI~O}cIhT+(vSq%O{}>gTrD)qxHL6) zqczm3Q|0k-n*$tA>k=zVl=_9&WS!(i^&va0Hq%;AOi%9U&-h1HiH2#~9d~;lX2BL7 z2^ynyF-_=({Qh)?&}(t?-#k9+j>Z>MD`IL+luA<_vP}Rr`^v~9XIWP7%;DttJFge3 z{Pj=Y>lQ3T%MicVEv4E(df-%#Ttl|Mn~{?91wCXF^`(w%!;`hge+n$us#VIGIT=RaEZP>S?yn2(oFM+YTHzeDC)$LUo03XPh`& zaY=4)pPI5oTqrS_tU=N=Y@0JEKQ*Q@5NaLuE!Sg7oRknpVv88YwueHFsYlX{Xa|c8 zZJ(WZ(?)oe{A@lyP;%F;XZ#4NKRMyPAZa9^D}2?`u5Z2?pjznDdN;(CNv1PRxUe&6 zMtrog@)T8e*|Bo+)8QB3&Qna&*o4TDrk)oMs&_m*U*5(aD^T{bn?{jxa$_0VL`}|4 za~62&Jy%MXKvw>uL&%Y&?Nq#S6GSG3eiI+e26@p>ffvOT_wXTsFfCi1KEWTa1(-P( zXxFu34)vbN0#aeSokG@rjhBf>{=wz1gO4W_UVf&-Tg_iD7f`iGs#_-PpgLZmt~o^R zxbzqDj^W%CK9nkL?-#5rcnPzx*Z3%oLAEn>3JmxXsRdQHH=8!1m0t9jLu4XxA4XZq%lJ?@kK$6e1^nX-asi*XPpSMYEUoF1^Rc zYnRv;<$6?}pYPWkmn1JQ{YV-%?(=ex20sYwYd_G})&tGvF(9q@ObGCN4tE7=+Q-i< z1fZjSr||_3L0x?|jg00Qys3SaCf~0lxTG$!w}1A2e=+U6?pw~ott|htrdP3zbK(1l zyH{e5H@)C}#w`IV3ZmxuH4KbBu^T%@7l1Is!Pi+{Ep$k9m(FaeXlqOn_D4%n_)JHd z|BDm#BpGc$;E!Fx*<{%GSU?z%2gnX&dxlPD*#nTzf3~p8ut8aFIlDO^WR#FdMfuBs zoGemF2EakVfA&yOX5k=v01mQ60S2AH%HIn3f0^}8F)F^JXT<#-Zd%I4gMxtNQTqD- zn6>Zv{6;^i>o;zMYb|u^`;8DB#B%^Glgrk4SIux0lB)-J377j$!hA=s=#1)OrXZ~^;?{~-8%ILq?}RF-J4?LM($2@ za*PMl6(dt3)c2?1m*aFll)9 zh3p|_3Z#i#c>PQoLKnko8=6GCeXdqVjf=&gq;JMIyVQlCw$;4GMPBeLTf^jE?#ebr z@uh$*?dj9dcR=#wjoR(_U-~~uX(E12Bbi;(rX9$(=A^uL~7wCZbow1+% zV`>3Obu$ex5>5~ZHJLHzmt71U)_yOEY1eKQf`-vVKGkG}q?HNVT^FQ5Up>j1VM?u* z;uU#rIBFXxB&J|$I<4;GRQB_fdbYCR+6TmIbfk52ek$~~?-04LvB8#Yi<&<_a)Wpj z8B)|7aCF@X`^4CL)4({#J<1Fva>}Pml&%l95Tfp|6Jrg)8doPKy@`*%^H+Ycwnjog zP}KGKCtnm}9W6Afi0bQ@=`il%3X2Pa3Z>);#o~$Uwp7Oy#@uG8F(HZI^JWx{L9-;K zQ6#t{vw^vb+T4u~R;pt8ah`r>Q+W>tOqZ-(J1kWwSHpi6v5^q-bz-zmmY2>7-}YM| z-w&35NN%`XRCascr&R7IGS_=EagIqVOY?c0=rW;K)n#3AVyg>*O8dqM#7o~RqLnG6 zObm`mL3U=rtQ(&eJ$_M(4iZsSF|5A14qgcl$05J3>o3N;5#CHHovY$;LohunO3oNrstEL!(N9@^(ZHa)Yt8$<~?&-)c#i;G)J@10JQZ zUKz0qcIo(@?)(`W7Xlo^I6CADIM0rHnm+p)A?lPqE?lj8N;6QiZh!1r+Ag8(oKb`4 z8MG}}V-e55h`88cVLs%-%D0U3b)&kscbhJm1nI{plmOWf8EX-8Vxx?gSRXP+}%?Ty}8 m*~F70i{q%=p=^{KRku5fZ)|nqjy&p*J^D>ELrnj9^8Wy?KDQ$P literal 0 HcmV?d00001 diff --git a/core/src/test/resources/ssl/otherhost.p12 b/core/src/test/resources/ssl/otherhost.p12 new file mode 100644 index 0000000000000000000000000000000000000000..d9543037ea52ac1c7c235cffe715d193c7b9207d GIT binary patch literal 2622 zcma)8XE+;-7LG_n5Uci9BOyg1wD_o1pH)@6D6vJOMvYJ@NvzapscNmF_GkCA)hL2m zEgeRwS~Y6V()zgWbMMpd-e33rIOja)J?DL&^Xq*Mio&D@q@zbspbHF8*%*VE?=U(D zT`mRs5k!HGoWVmV3b^*aM&Lpa1^oOBes&h=49x##u`tmAb17i0Gf*An@rwb5p=?nq zf43|sB#29bKV}}&4zq?5(@sL0`Mhm5x2ZrNoE=CfhGJo0{FfpS3I?Er8K6!v26XQ9 zAUatPd$|jmt73c*FlU%bh@Bb!7_qlyBS^!fS{R`; zYp?9wec7i!U^|$oR_WD}{78at_g0JI+)AImnRL)K1Jg#G{1>L&C(^IfMdzzG2 z?@!h$fsDkr7I~Tmli#Lh4><546NmS3Qc6AX7n0?A$mV*rIZ}xRH)r3COk}RZ_=j**`ESmSE$W?rTYE{vu06hNIr{Z%&xE9l^yS10j zDOKIv>SQIJ3okM1(yS;)fXr?Pqe}bl$sOB#xIT#8<)q#OKkP3hPVSW3Bo%*Xfap@rd!>R?nPz?Ry`&M9jKsF1T9Fm)VV=`V@a_zQbLW&AyG|!hvQ|RBhdY_7ajj(iGee$ z!WTwg9^APk6gTqy)>~ESVf;=>N6K7is{hjA;>C0&`j)(@Vh7nn19UN@A*)-zC7~M- zWdqXov9yt{7aW$9LoSv|&>GXVO-}Ho`%nOZwq2I*P6t|!F9^P)sqF?Ioz|XJ`P7WQ z>(>7Re0-}V4o*YmHekXe`MU(ZJvy-^W?)Tu`-jX1&2lV5@`_x+NGva69tT%592<>k z{F0m8RMmWJZ8$4jo?>T!lX;Al)>wNWHnlyW6G#|odQ&ax$PP%`1X2n58pC&u60Y|y z>XoZ50%ND*t`tj5>#+Xy=kK=pdG?nBWqDb_Cfu3=HaDNx4?H;D<#IG`F?-1AfiO5I z^|BK1&#%bhnUf!Q@A1!r2zH%pT@80e3UHd?xnpto~$yIW-8G0~WOE^;~jiv$VO z+>uk1V$_xHjeAaX&y4XPFFnQDrTbLFR)eAjJ|x|Y(V*lnkr7U~9rhzNlrqLkA0Y89*b5vPjoAZ}TS(Mg|IjW-3x zo{|X;2ZP1!Zy6a>uikj~MhzE{f?hQ|aE(Xo51qF4;XQH{IAThjJg0W}>uL_y&&k>H z$=>m-M#hS~1AAXoiU?s@+0XS}n{z4inaSvqlhfBc-1gz=cwLed`DqRZa7c4ETe8(! zm9O7on&b*EnXAE-cXQppJ~VIL5p4#Kv-k8r3BM2#ZC&fp?^J<*YOl57uo@X=p;M(< zMsqZ;(>gtI4-~+^LBq4HQ(baIJ5S`Mcirr?cMMv`Y9e>|$u#jKn|adY?kE2d+iB7> z$o=q0ArSg^l=v^ZgmR&|L^uJyfIt8a;10NRmVE&MfWZG;;YM?TI4!+yyNjVOsi<63 zQbl1f$`}la0^<0k17XOefPg3p{Rxl`a8|zmJ)!$wWkpKT*eK@ZXE*r{L$(2*3||Bd*K?HDN<=D0krVIJk|yZVm=_M~Bj$%;pDQ#)pK-I&XUPUx;M2$+KxtZJ${#w0#Z0r<~tck;O=SY~hym<4|_w zOin8rvgZ_Sa0r>_gW7I(*l3+{`cu4xPmfdRd0{aNnQQp6j++sNtKRCJtW;u7Vf`&) zc93N2cWX)fWuv9+(#qxhqC5ymYV0$PCmG<<7y%K4d^9a}T{}D`T5hNd9Mc4tFV;V< zv2g5trP++Vx$UP+MMSu81kOI4dHsDRzqW()l@-D5!#;7n|NWq@PP=Mz<0EGOcmAZ2 z2T~Tm7ums{Knzdkl<%=&LmZ#lH!?Pc#fNoPjWlQ?{Zz5)5?!l8rOq^r!8Se=;f%j? zYSr*?sYKWE_-J(QC)L-vq~d*PdsXk&li^0I;#^uGd?$m-=d+N-{r3;U$_;^mD?tgNiVDhpqP`%aqlswLLtcyjTf z*Mh7AZkH&A=*i4xPizg=LHin5;;Mi4{@HLxXt;yIrN z+A{s-#aSs4DpYfk=s}&cwafT3{#iuzV})7S`?nXqRiv~(_`$od9akA}G`igHC-Nd| z{2;Fi?~huO`i&`MDyUW`Bw0o&depS${n%kcS|WV)ucmZ&5C|o%)d2oEiRNH}F0h$- zW!TmsE+Oz(vqbi?r60)~lUj<~I(w#c6KiQl0Rd zC9w0sFtF%6z21%%Rmf3gAl``GWO_kYOc49+5#LLDCKje2C=(PK1^e|=0qMX1kQhU; z*d?xdFv}GM%|+}-t#^eX5?4S-lVp7CMO`S>W}gQkMxKl{?d{w9t3P>_&q_t Ie_Fsl055K*s{jB1 literal 0 HcmV?d00001 diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc index c57fca469..75e77668b 100644 --- a/docs/src/main/asciidoc/configuration.adoc +++ b/docs/src/main/asciidoc/configuration.adoc @@ -261,7 +261,9 @@ header. | http.content.partial.as.trimmed | false | Accepts partially fetched content in OKHTTP. | http.filter.ipaddress.include | - | (OkHttp only) Comma-separated list (or YAML list) of allowed IP ranges. If empty, all addresses are allowed unless excluded. See <>. | http.filter.ipaddress.exclude | - | (OkHttp only) Comma-separated list (or YAML list) of blocked IP ranges. See <>. -| http.trust.everything | true | If true, trust all SSL/TLS connections. +| http.trust.everything | false | (OkHttp only) If true, accept any TLS certificate, including self-signed, expired or otherwise invalid ones. The servers are then not authenticated: anyone able to answer for the host name receives everything sent to them. Credentials (basic auth, credential headers, cookies) are withheld over such connections unless `http.credentials.allow.insecure` is set to true. +| http.verify.hostnames | true | (OkHttp only) If true, check that the certificate presented by the server matches the host name contacted. Independent of `http.trust.everything`: a valid certificate for a different name is accepted when this is false. +| http.credentials.allow.insecure | false | (OkHttp only) If true, send credentials (basic auth, credential headers, cookies) even over connections whose server certificate was not validated, i.e. when `http.trust.everything` is true. | topology.message.timeout.secs | -1 | OKHTTP message timeout. |=== diff --git a/docs/src/main/asciidoc/extending.adoc b/docs/src/main/asciidoc/extending.adoc index 0837db0a7..a22494295 100644 --- a/docs/src/main/asciidoc/extending.adoc +++ b/docs/src/main/asciidoc/extending.adoc @@ -392,7 +392,7 @@ Keep the pool size (idle + active) under 1000. For large crawls, increase `proto ==== SSL/TLS -By default, StormCrawler's OkHttp protocol trusts all certificates (`http.trust.everything: true`). For production crawls over untrusted networks, consider setting this to `false` and configuring a proper trust store. +By default, StormCrawler's OkHttp protocol validates the certificate chains of HTTPS servers and checks that the certificate matches the host name contacted (`http.trust.everything: false`, `http.verify.hostnames: true`). For crawls of hosts whose certificates cannot be validated, e.g. self-signed ones, set `http.trust.everything` to `true`: the servers are then not authenticated, so credentials (basic auth, credential headers in `http.custom.headers`, cookies) are withheld over such connections unless `http.credentials.allow.insecure` is set to `true`. Hostname checking is a separate decision and can be disabled with `http.verify.hostnames: false`. ==== Authentication @@ -404,7 +404,7 @@ http.basicauth.user: "username" http.basicauth.password: "password" ---- -These credentials are sent as an `Authorization` header with every request. For per-site authentication, use metadata-driven headers instead (see <> in the Internals section). +These credentials are sent as an `Authorization` header with every request to servers authenticated by their TLS certificate; they are withheld when `http.trust.everything` is enabled, unless `http.credentials.allow.insecure` is set to `true`. For per-site authentication, use metadata-driven headers instead (see <> in the Internals section). ==== Proxy Authentication diff --git a/external/opensearch/archetype/src/main/resources/archetype-resources/crawler-conf.yaml b/external/opensearch/archetype/src/main/resources/archetype-resources/crawler-conf.yaml index f62103faf..f2d9860f5 100644 --- a/external/opensearch/archetype/src/main/resources/archetype-resources/crawler-conf.yaml +++ b/external/opensearch/archetype/src/main/resources/archetype-resources/crawler-conf.yaml @@ -72,6 +72,12 @@ config: http.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol" https.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol" + # Accept any TLS certificate, e.g. self-signed ones? The servers are not + # authenticated then; credentials (basic auth, credential headers, cookies) + # are withheld over such connections unless + # http.credentials.allow.insecure is set to true. + http.trust.everything: false + # The maximum number of bytes for returned HTTP response bodies. # The fetched page will be trimmed to 65KB in this case # Set -1 to disable the limit. diff --git a/external/solr/archetype/src/main/resources/archetype-resources/crawler-conf.yaml b/external/solr/archetype/src/main/resources/archetype-resources/crawler-conf.yaml index f62103faf..f2d9860f5 100644 --- a/external/solr/archetype/src/main/resources/archetype-resources/crawler-conf.yaml +++ b/external/solr/archetype/src/main/resources/archetype-resources/crawler-conf.yaml @@ -72,6 +72,12 @@ config: http.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol" https.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol" + # Accept any TLS certificate, e.g. self-signed ones? The servers are not + # authenticated then; credentials (basic auth, credential headers, cookies) + # are withheld over such connections unless + # http.credentials.allow.insecure is set to true. + http.trust.everything: false + # The maximum number of bytes for returned HTTP response bodies. # The fetched page will be trimmed to 65KB in this case # Set -1 to disable the limit. From e9c0a618b49059e985d866fa8617cccaf0173576 Mon Sep 17 00:00:00 2001 From: abhinav-phi Date: Sun, 6 Sep 2026 13:15:22 +0530 Subject: [PATCH 2/7] OkHttpTrustEverythingTest: serve TLS with WireMock The JDK's com.sun.net.httpserver classes are rejected by the forbiddenapis check (jdk-non-portable). WireMock is already a test dependency and serves HTTPS from a PKCS12 keystore; the request journal replaces the hand-written header recorder. The custom headers use the Name=value syntax that http.custom.headers parses and the code is google-java-format clean. --- .../okhttp/OkHttpTrustEverythingTest.java | 155 ++++++++---------- 1 file changed, 66 insertions(+), 89 deletions(-) diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java index 82d1c136c..3ff51de76 100644 --- a/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java +++ b/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java @@ -17,27 +17,27 @@ package org.apache.stormcrawler.protocol.okhttp; +import static com.github.tomakehurst.wiremock.client.WireMock.any; +import static com.github.tomakehurst.wiremock.client.WireMock.anyUrl; +import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; +import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.ok; +import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; -import com.sun.net.httpserver.HttpExchange; -import com.sun.net.httpserver.HttpHandler; -import com.sun.net.httpserver.HttpsConfigurator; -import com.sun.net.httpserver.HttpsServer; -import java.io.IOException; +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.common.ConsoleNotifier; +import com.github.tomakehurst.wiremock.core.WireMockConfiguration; import java.io.InputStream; -import java.io.OutputStream; -import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; -import java.security.KeyStore; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.util.Base64; import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import javax.net.ssl.KeyManagerFactory; -import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLHandshakeException; +import javax.net.ssl.SSLPeerUnverifiedException; import org.apache.storm.Config; import org.apache.stormcrawler.Metadata; import org.apache.stormcrawler.protocol.ProtocolResponse; @@ -51,7 +51,7 @@ */ class OkHttpTrustEverythingTest { - private static final char[] KEYSTORE_PASSWORD = "changeit".toCharArray(); + private static final String KEYSTORE_PASSWORD = "changeit"; /** Certificate issued for localhost: valid for the host the tests connect to. */ private static final String LOCALHOST_KEYSTORE = "/ssl/localhost.p12"; @@ -59,16 +59,13 @@ class OkHttpTrustEverythingTest { /** Certificate issued for another host name: trusted under trust-all, wrong name. */ private static final String OTHERHOST_KEYSTORE = "/ssl/otherhost.p12"; - private HttpsServer server; - - private RecordingHandler handler; + private WireMockServer server; @AfterEach void stopServer() { if (server != null) { - server.stop(0); + server.stop(); server = null; - handler = null; } } @@ -83,12 +80,11 @@ void trustAllContextUsesTls() { @Test void selfSignedCertificateRejectedByDefault() throws Exception { // http.trust.everything defaults to false: an unvalidatable certificate - // must not be accepted. Depending on the platform the handshake fails or - // the connection is dropped while it is retried. + // must not be accepted startServer(LOCALHOST_KEYSTORE); final HttpProtocol protocol = protocol(config()); assertThrows( - Exception.class, + SSLHandshakeException.class, () -> fetch(protocol, "/default"), "self-signed certificates must be rejected by default"); } @@ -105,14 +101,12 @@ void trustAllFetchesServerWithSelfSignedCertificate() throws Exception { @Test void hostnameIsStillVerified() throws Exception { // the certificate is issued for another host name: trusting any - // certificate must not imply accepting any name. Depending on the - // platform the failed verification surfaces as an SSL exception or the - // connection is dropped while it is retried. + // certificate must not imply accepting any name final Config conf = config(); conf.put("http.trust.everything", true); startServer(OTHERHOST_KEYSTORE); assertThrows( - Exception.class, + SSLPeerUnverifiedException.class, () -> fetch(protocol(conf), "/hostname"), "the hostname verifier should not accept any name unconditionally"); } @@ -134,11 +128,9 @@ void basicAuthIsWithheldFromUnauthenticatedServers() throws Exception { conf.put("http.basicauth.user", "user"); conf.put("http.basicauth.password", "secret"); startServer(LOCALHOST_KEYSTORE); - final ProtocolResponse response = fetch(protocol(conf), "/basicauth"); - assertEquals(200, response.getStatusCode(), "the connection must succeed"); - assertNull( - handler.lastHeaders.get("authorization"), - "credentials must not be sent to unauthenticated servers"); + fetch(protocol(conf), "/basicauth"); + server.verify( + 1, getRequestedFor(urlPathEqualTo("/basicauth")).withoutHeader("Authorization")); } @Test @@ -154,10 +146,10 @@ void basicAuthIsSentWhenExplicitlyAllowed() throws Exception { "Basic " + Base64.getEncoder() .encodeToString("user:secret".getBytes(StandardCharsets.UTF_8)); - assertEquals( - expected, - handler.lastHeaders.get("authorization"), - "the opt-in sends credentials"); + server.verify( + 1, + getRequestedFor(urlPathEqualTo("/basicauth")) + .withHeader("Authorization", equalTo(expected))); } @Test @@ -166,11 +158,13 @@ void credentialCustomHeadersAreWithheldFromUnauthenticatedServers() throws Excep conf.put("http.trust.everything", true); conf.put("http.custom.headers", List.of("X-Api-Key=s3cret", "X-Trace=public")); startServer(LOCALHOST_KEYSTORE); - final ProtocolResponse response = fetch(protocol(conf), "/customheaders"); - assertEquals(200, response.getStatusCode(), "the connection must succeed"); - assertNull( - handler.lastHeaders.get("x-api-key"), "credential headers must be withheld"); - assertEquals("public", handler.lastHeaders.get("x-trace"), "other headers are sent"); + fetch(protocol(conf), "/customheaders"); + server.verify( + 1, getRequestedFor(urlPathEqualTo("/customheaders")).withoutHeader("X-Api-Key")); + server.verify( + 1, + getRequestedFor(urlPathEqualTo("/customheaders")) + .withHeader("X-Trace", equalTo("public"))); } @Test @@ -181,8 +175,10 @@ void credentialCustomHeadersAreSentWhenExplicitlyAllowed() throws Exception { conf.put("http.custom.headers", List.of("X-Api-Key=s3cret", "X-Trace=public")); startServer(LOCALHOST_KEYSTORE); fetch(protocol(conf), "/customheaders"); - assertEquals("s3cret", handler.lastHeaders.get("x-api-key"), "the opt-in sends credentials"); - assertEquals("public", handler.lastHeaders.get("x-trace")); + server.verify( + 1, + getRequestedFor(urlPathEqualTo("/customheaders")) + .withHeader("X-Api-Key", equalTo("s3cret"))); } @Test @@ -191,11 +187,8 @@ void cookiesAreWithheldFromUnauthenticatedServers() throws Exception { conf.put("http.trust.everything", true); conf.put("http.use.cookies", true); startServer(LOCALHOST_KEYSTORE); - final ProtocolResponse response = fetch(protocol(conf), "/cookies", metadata()); - assertEquals(200, response.getStatusCode(), "the connection must succeed"); - assertNull( - handler.lastHeaders.get("cookie"), - "cookies must not be sent to unauthenticated servers"); + fetch(protocol(conf), "/cookies", metadata()); + server.verify(1, getRequestedFor(urlPathEqualTo("/cookies")).withoutHeader("Cookie")); } @Test @@ -206,16 +199,16 @@ void cookiesAreSentWhenExplicitlyAllowed() throws Exception { conf.put("http.use.cookies", true); startServer(LOCALHOST_KEYSTORE); fetch(protocol(conf), "/cookies", metadata()); - assertEquals("sid=x", handler.lastHeaders.get("cookie"), "the opt-in sends cookies"); + server.verify( + 1, + getRequestedFor(urlPathEqualTo("/cookies")).withHeader("Cookie", equalTo("sid=x"))); } /** Metadata as an outlink would inherit it, with a cookie scoped to the server. */ private Metadata metadata() { final Metadata md = new Metadata(); md.setValue("protocol.set-cookie", "sid=x; Path=/"); - md.setValue( - "protocol.set-cookie-origin", - "https://localhost:" + server.getAddress().getPort() + "/"); + md.setValue("protocol.set-cookie-origin", "https://localhost:" + server.httpsPort() + "/"); return md; } @@ -242,48 +235,32 @@ private ProtocolResponse fetch(HttpProtocol protocol, String path) throws Except private ProtocolResponse fetch(HttpProtocol protocol, String path, Metadata md) throws Exception { - return protocol.getProtocolOutput( - "https://localhost:" + server.getAddress().getPort() + path, md); + return protocol.getProtocolOutput("https://localhost:" + server.httpsPort() + path, md); } - /** Starts an HTTPS server on a random port presenting the certificate of the keystore. */ + /** + * Starts an HTTPS server on a random port presenting the certificate of the keystore. The + * keystore is copied to a temporary file as WireMock reads it from the file system. + */ private void startServer(String keystoreResource) throws Exception { - final KeyStore keyStore = KeyStore.getInstance("PKCS12"); - try (InputStream in = OkHttpTrustEverythingTest.class.getResourceAsStream(keystoreResource)) { - keyStore.load(in, KEYSTORE_PASSWORD); + final Path keystoreFile = Files.createTempFile("wiremock-keystore", ".p12"); + keystoreFile.toFile().deleteOnExit(); + try (InputStream in = + OkHttpTrustEverythingTest.class.getResourceAsStream(keystoreResource)) { + Files.copy(in, keystoreFile, StandardCopyOption.REPLACE_EXISTING); } - final KeyManagerFactory keyManagerFactory = - KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); - keyManagerFactory.init(keyStore, KEYSTORE_PASSWORD); - final SSLContext sslContext = SSLContext.getInstance("TLS"); - sslContext.init(keyManagerFactory.getKeyManagers(), null, null); - server = HttpsServer.create(new InetSocketAddress(java.net.InetAddress.getByName("127.0.0.1"), 0), 0); - server.setHttpsConfigurator(new HttpsConfigurator(sslContext)); - handler = new RecordingHandler(); - server.createContext("/", handler); + server = + new WireMockServer( + WireMockConfiguration.options() + .dynamicPort() + .dynamicHttpsPort() + .keystorePath(keystoreFile.toAbsolutePath().toString()) + .keystorePassword(KEYSTORE_PASSWORD) + .keyManagerPassword(KEYSTORE_PASSWORD) + .keystoreType("PKCS12") + .notifier(new ConsoleNotifier(false))); server.start(); - } - - /** Records the request headers of the last request received. */ - static class RecordingHandler implements HttpHandler { - - final Map lastHeaders = new ConcurrentHashMap<>(); - - @Override - public void handle(HttpExchange exchange) throws IOException { - for (Map.Entry> header : - exchange.getRequestHeaders().entrySet()) { - lastHeaders.put( - header.getKey().toLowerCase(Locale.ROOT), - String.join(",", header.getValue())); - } - final byte[] body = "Success!".getBytes(StandardCharsets.UTF_8); - exchange.getResponseHeaders().set("Content-Type", "text/html"); - exchange.sendResponseHeaders(200, body.length); - try (OutputStream out = exchange.getResponseBody()) { - out.write(body); - } - } + server.stubFor(any(anyUrl()).willReturn(ok("Success!"))); } } From 68371df0f25e1bd7c7493fffcc2bf6426529b06b Mon Sep 17 00:00:00 2001 From: abhinav-phi Date: Mon, 7 Sep 2026 23:04:30 +0530 Subject: [PATCH 3/7] Withhold credentials from every unauthenticated server (#2079) Review feedback on #2126: - the decision to send credentials is now made per request: a server is only authenticated when the connection is https:// and the certificate chain is validated. Cleartext http:// requests never authenticate the server and no longer receive the Authorization header, credential headers or cookies, closing the cleartext case the first commit missed - the credential header denylist is configurable with http.credentials.headers (default: authorization, proxy-authorization, cookie, x-api-key), so site-specific names like x-auth-token can be covered without dropping every custom header - the withheld-cookies warning only fires when a cookie was actually going to be sent, and the withheld-request-headers warning is logged once per protocol instance instead of once per header per request - the proxy authenticator keeps its credentials by design and says so in a comment: the proxy is not the crawled server - the keytool commands that generated the test keystores are recorded in the test class, and the TLS test server moved to WireMock so the forbiddenapis check passes --- .../protocol/okhttp/HttpProtocol.java | 192 ++++++++++++------ core/src/main/resources/crawler-default.yaml | 25 ++- .../HttpRobotRulesParserRedirectTest.java | 3 + .../okhttp/HttpProtocolCookieTest.java | 3 + .../okhttp/OkHttpTrustEverythingTest.java | 75 ++++++- docs/src/main/asciidoc/configuration.adoc | 5 +- docs/src/main/asciidoc/extending.adoc | 4 +- 7 files changed, 235 insertions(+), 72 deletions(-) diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java index 71183a58a..17194fdd1 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java @@ -28,6 +28,7 @@ import java.security.cert.CertificateException; import java.util.ArrayList; import java.util.Base64; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Locale; @@ -110,13 +111,11 @@ public class HttpProtocol extends AbstractHttpProtocol { // makes sure that withheld cookies are reported once and not for every url private final AtomicBoolean withheldCookiesLogged = new AtomicBoolean(); - // whether credentials (basic auth, credential headers, cookies) may be sent: - // false when the servers are not authenticated and - // http.credentials.allow.insecure is not enabled - private boolean sendCredentials = true; + // makes sure that withheld request headers are reported once and not for every url + private final AtomicBoolean withheldRequestHeadersLogged = new AtomicBoolean(); - /** Header names carrying credentials, see {@link #isCredentialHeader(String)}. */ - private static final Set CREDENTIAL_HEADERS = + /** Default for http.credentials.headers: header names (lower case) carrying credentials. */ + private static final Set DEFAULT_CREDENTIAL_HEADERS = Set.of( HttpHeaders.AUTHORIZATION.toLowerCase(Locale.ROOT), HttpHeaders.PROXY_AUTHORIZATION.toLowerCase(Locale.ROOT), @@ -124,6 +123,22 @@ public class HttpProtocol extends AbstractHttpProtocol { "cookie", "x-api-key"); + // header names (lower case) considered to carry credentials, configured with + // http.credentials.headers, see #isCredentialHeader + private Set credentialHeaders = DEFAULT_CREDENTIAL_HEADERS; + + // request headers which carry credentials; they are only sent to servers + // which were authenticated, see #credentialsAllowed + private final List credentialRequestHeaders = new LinkedList<>(); + + // accept any certificate chain (http.trust.everything): https connections + // do not authenticate the servers then either + private boolean trustEverything = false; + + // send credentials even to servers which were not authenticated + // (http.credentials.allow.insecure) + private boolean insecureCredentialsAllowed = false; + private OkHttpClient.Builder builder; private static final TrustManager[] trustAllCerts = @@ -179,13 +194,26 @@ public void configure(Config conf) { * certificate trust and hostname verification are separate decisions: * accepting any certificate does not imply accepting any name */ - final boolean trustEverything = ConfUtils.getBoolean(conf, "http.trust.everything", false); + this.trustEverything = ConfUtils.getBoolean(conf, "http.trust.everything", false); final boolean verifyHostnames = ConfUtils.getBoolean(conf, "http.verify.hostnames", true); - // credentials are withheld over connections whose server certificate - // was not validated, unless explicitly opted in - final boolean insecureCredentialsAllowed = + // credentials are withheld on connections which do not authenticate the + // server, unless explicitly opted in + this.insecureCredentialsAllowed = ConfUtils.getBoolean(conf, "http.credentials.allow.insecure", false); - this.sendCredentials = !trustEverything || insecureCredentialsAllowed; + + // header names considered to carry credentials; when set, + // http.credentials.headers replaces the default list + final List configuredCredentialHeaders = + ConfUtils.loadListFromConf("http.credentials.headers", conf); + if (!configuredCredentialHeaders.isEmpty()) { + final Set names = new HashSet<>(); + for (String name : configuredCredentialHeaders) { + if (StringUtils.isNotBlank(name)) { + names.add(name.trim().toLowerCase(Locale.ROOT)); + } + } + credentialHeaders = names; + } if (trustEverything) { LOG.warn( @@ -198,13 +226,6 @@ public void configure(Config conf) { "http.verify.hostnames is disabled: the certificates are not checked against " + "the host name either, the identity of the servers is not authenticated."); } - if (trustEverything && !sendCredentials) { - LOG.warn( - "Credentials configured with http.basicauth.*, credential headers in " - + "http.custom.headers and cookies are withheld because the servers " - + "are not authenticated. Set http.credentials.allow.insecure to true " - + "to send them anyway."); - } builder = new OkHttpClient.Builder() @@ -268,35 +289,43 @@ public void configure(Config conf) { final String basicAuthUser = ConfUtils.getString(conf, "http.basicauth.user", null); - // use a basic auth? + // use a basic auth? the Authorization header carries credentials and is + // only sent to servers which were authenticated, see #credentialsAllowed if (StringUtils.isNotBlank(basicAuthUser)) { final String basicAuthPass = ConfUtils.getString(conf, "http.basicauth.password", ""); - if (sendCredentials) { - final String encoding = - Base64.getEncoder() - .encodeToString( - (basicAuthUser + ":" + basicAuthPass) - .getBytes(StandardCharsets.UTF_8)); - customRequestHeaders.add( - new KeyValue(HttpHeaders.AUTHORIZATION, "Basic " + encoding)); + final String encoding = + Base64.getEncoder() + .encodeToString( + (basicAuthUser + ":" + basicAuthPass) + .getBytes(StandardCharsets.UTF_8)); + credentialRequestHeaders.add( + new KeyValue(HttpHeaders.AUTHORIZATION, "Basic " + encoding)); + } + + for (KeyValue customHeader : customHeaders) { + if (isCredentialHeader(customHeader.getKey())) { + credentialRequestHeaders.add(customHeader); } else { - LOG.warn( - "Basic authentication configured with http.basicauth.user is withheld " - + "because the servers are not authenticated (http.trust.everything). " - + "Set http.credentials.allow.insecure to true to send it anyway."); + customRequestHeaders.add(customHeader); } } - for (KeyValue customHeader : customHeaders) { - if (!sendCredentials && isCredentialHeader(customHeader.getKey())) { + if (!credentialRequestHeaders.isEmpty() || useCookies) { + if (trustEverything && !insecureCredentialsAllowed) { LOG.warn( - "Custom header {} is withheld because the servers are not authenticated " - + "(http.trust.everything). Set http.credentials.allow.insecure to " - + "true to send it anyway.", - customHeader.getKey()); - continue; + "Credentials configured with http.basicauth.*, credential headers in " + + "http.custom.headers and cookies are withheld from every request " + + "because the servers are not authenticated (http.trust.everything). " + + "Set http.credentials.allow.insecure to true to send them anyway."); + } else if (!insecureCredentialsAllowed) { + LOG.info( + "Credentials configured with http.basicauth.*, credential headers in " + + "http.custom.headers and cookies are sent only over https:// urls " + + "with certificate validation. They are withheld from cleartext " + + "http:// urls and from https:// urls when http.trust.everything " + + "is enabled; set http.credentials.allow.insecure to true to send " + + "them there anyway."); } - customRequestHeaders.add(customHeader); } // optionally block connections to forbidden IP address ranges @@ -346,21 +375,20 @@ public EventListener create(Call call) { client = builder.build(); } - private void addCookiesToRequest(Builder rb, String url, Metadata md) { + private void addCookiesToRequest(Builder rb, String url, Metadata md, boolean sendCredentials) { + final String[] cookieStrings = + md.getValues(RESPONSE_COOKIES_HEADER, protocolMetadataPrefix); + if (cookieStrings == null || cookieStrings.length == 0) { + return; + } if (!sendCredentials) { if (withheldCookiesLogged.compareAndSet(false, true)) { LOG.warn( - "Cookies are withheld because the servers are not authenticated " - + "(http.trust.everything). Set http.credentials.allow.insecure to " - + "true to send them anyway."); + "Cookies are withheld because the server is not authenticated. Set " + + "http.credentials.allow.insecure to true to send them anyway."); } return; } - final String[] cookieStrings = - md.getValues(RESPONSE_COOKIES_HEADER, protocolMetadataPrefix); - if (cookieStrings == null || cookieStrings.length == 0) { - return; - } try { final List cookies = CookieConverter.getCookies( @@ -410,29 +438,49 @@ private URL getCookieOrigin(Metadata md, String url) { /** * Returns true when the header carries credentials which must not be disclosed to servers that - * were not authenticated. Covers the standard credential headers plus any header commonly used - * for API keys; cookie headers are handled separately in {@link #addCookiesToRequest}. + * were not authenticated. The names are configured with http.credentials.headers and default to + * the standard credential headers plus any header commonly used for API keys. */ - private static boolean isCredentialHeader(String name) { + private boolean isCredentialHeader(String name) { if (name == null) { return false; } final String normalised = name.trim().toLowerCase(Locale.ROOT); - return CREDENTIAL_HEADERS.contains(normalised); + return credentialHeaders.contains(normalised); + } + + /** + * Returns true when credentials (basic auth, credential headers, cookies) may be sent with the + * request to the url. A server is only authenticated when the connection is HTTPS and the + * certificate chain is validated; on cleartext http:// connections and on https:// connections + * which accept any certificate (http.trust.everything), credentials are withheld unless + * http.credentials.allow.insecure is enabled. + */ + private boolean credentialsAllowed(String url) { + if (insecureCredentialsAllowed) { + return true; + } + try { + return URLUtil.toURL(url).getProtocol().equals("https") && !trustEverything; + } catch (MalformedURLException e) { + return false; + } } - protected void addHeadersToRequest(Builder rb, Metadata md) { + protected void addHeadersToRequest(Builder rb, Metadata md, boolean sendCredentials) { final String[] headerStrings = md.getValues(SET_HEADER_BY_REQUEST, protocolMetadataPrefix); if (headerStrings != null && headerStrings.length > 0) { for (String hs : headerStrings) { KeyValue h = KeyValue.build(hs); if (!sendCredentials && isCredentialHeader(h.getKey())) { - LOG.warn( - "Header {} set by request is withheld because the servers are not " - + "authenticated (http.trust.everything). Set " - + "http.credentials.allow.insecure to true to send it anyway.", - h.getKey()); + if (withheldRequestHeadersLogged.compareAndSet(false, true)) { + LOG.warn( + "Header {} set by request is withheld because the server is not " + + "authenticated. Set http.credentials.allow.insecure to " + + "true to send it anyway.", + h.getKey()); + } continue; } rb.addHeader(h.getKey(), h.getValue()); @@ -471,7 +519,10 @@ public ProtocolResponse getProtocolOutput(String url, final Metadata metadata) // conditionally add proxy authentication if (StringUtils.isNotBlank(prox.getUsername())) { - // add proxy authentication header to builder + // the proxy is not the crawled server: its credentials + // authenticate against the proxy itself and are sent + // regardless of whether the target server was + // authenticated, cf. #credentialsAllowed localBuilder.proxyAuthenticator( (Route route, Response response) -> { String credential = @@ -499,16 +550,35 @@ public ProtocolResponse getProtocolOutput(String url, final Metadata metadata) } } + // credentials are only sent to servers which were authenticated, which + // depends on the url of the individual request + final boolean sendCredentials = credentialsAllowed(url); + final Builder rb = new Request.Builder().url(url); customRequestHeaders.forEach( (k) -> { rb.header(k.getKey(), k.getValue()); }); + if (sendCredentials) { + credentialRequestHeaders.forEach( + (k) -> { + rb.header(k.getKey(), k.getValue()); + }); + } else if (!credentialRequestHeaders.isEmpty() + && withheldRequestHeadersLogged.compareAndSet(false, true)) { + LOG.warn( + "Configured credential headers (http.basicauth.*, http.custom.headers) are " + + "withheld because {} is not authenticated. Set " + + "http.credentials.allow.insecure to true to send them anyway.", + url.startsWith("https") + ? "https with http.trust.everything" + : "cleartext http"); + } int pageMaxContent = globalMaxContent; if (metadata != null) { - addHeadersToRequest(rb, metadata); + addHeadersToRequest(rb, metadata, sendCredentials); final String lastModified = metadata.getFirstValue(HttpHeaders.LAST_MODIFIED); if (StringUtils.isNotBlank(lastModified)) { @@ -545,7 +615,7 @@ public ProtocolResponse getProtocolOutput(String url, final Metadata metadata) } if (useCookies) { - addCookiesToRequest(rb, url, metadata); + addCookiesToRequest(rb, url, metadata, sendCredentials); } final String postJsonData = metadata.getFirstValue("http.post.json"); diff --git a/core/src/main/resources/crawler-default.yaml b/core/src/main/resources/crawler-default.yaml index 273e90464..3e2a4b170 100644 --- a/core/src/main/resources/crawler-default.yaml +++ b/core/src/main/resources/crawler-default.yaml @@ -155,8 +155,9 @@ config: # answer for the host name receives everything sent to them. Needed for # crawling hosts with unvalidatable certificates, e.g. self-signed ones, but # also disables protection against man-in-the-middle attacks. Credentials - # (basic auth, credential headers, cookies) are withheld over such - # connections unless http.credentials.allow.insecure is set to true. + # (basic auth, credential headers, cookies) are withheld from cleartext + # http:// requests and from https:// requests on such connections unless + # http.credentials.allow.insecure is set to true. http.trust.everything: false # Check that the certificate presented by the server matches the host name @@ -166,13 +167,25 @@ config: # with self-signed certificates when http.trust.everything is enabled. http.verify.hostnames: true - # Send credentials (basic auth, credential headers, cookies) even over - # connections whose server certificate was not validated, i.e. when - # http.trust.everything is true (okhttp protocol only)? Without it, - # credentials are withheld so that they are not disclosed to servers which + # Send credentials (basic auth, credential headers, cookies) even on + # connections which do not authenticate the server (okhttp protocol only)? + # Without it, credentials are withheld from cleartext http:// requests and + # from https:// requests whose certificate was not validated + # (http.trust.everything), so that they are not disclosed to servers which # were never authenticated. http.credentials.allow.insecure: false + # Header names (case-insensitive) which carry credentials and are therefore + # withheld from requests to servers which were not authenticated (okhttp + # protocol only, see http.trust.everything and + # http.credentials.allow.insecure). Applies to the headers configured with + # http.custom.headers and to headers set by request; cookies are withheld + # separately. When set, this key replaces the default list of + # "authorization", "proxy-authorization", "cookie" and "x-api-key". + # http.credentials.headers: + # - "x-api-key" + # - "x-auth-token" + # IP address filtering (okhttp protocol only). Optionally limit or block # connections to IP address ranges once the host name has been resolved. This # prevents information leakage to a public index when a DNS entry points to a diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserRedirectTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserRedirectTest.java index 034489d65..19d7527ce 100644 --- a/core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserRedirectTest.java +++ b/core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserRedirectTest.java @@ -288,6 +288,9 @@ void testHeadersNotSentToAnUnfollowedRedirectTarget() { authConf.putAll(conf); authConf.put("http.basicauth.user", "this_is_only_a_test"); authConf.put("http.basicauth.password", "this_is_only_a_test"); + // the test verifies where the Authorization header is sent, not the + // withholding on unauthenticated connections: opt in + authConf.put("http.credentials.allow.insecure", true); HttpProtocol authProtocol = new HttpProtocol(); authProtocol.configure(authConf); try { diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocolCookieTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocolCookieTest.java index a30d52480..231d8bc2a 100644 --- a/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocolCookieTest.java +++ b/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocolCookieTest.java @@ -148,6 +148,9 @@ private HttpProtocol protocol(boolean useCookies) { conf.put("http.agent.url", "http://test.example.com"); conf.put("http.agent.email", "test@example.com"); conf.put("http.use.cookies", useCookies); + // the tests below verify cookie scoping mechanics, not the withholding + // of credentials on unauthenticated connections: opt in + conf.put("http.credentials.allow.insecure", true); conf.put("protocol.md.prefix", "protocol."); final HttpProtocol protocol = new HttpProtocol(); protocol.configure(conf); diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java index 3ff51de76..a0ff5707b 100644 --- a/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java +++ b/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java @@ -49,6 +49,18 @@ * configuration of the okhttp protocol must be an explicit choice, must not disable hostname * verification and must not disclose credentials to servers which are not authenticated. */ +/* + * The keystores under src/test/resources/ssl are self-signed PKCS12 keystores, generated with + * (JDK keytool, passwords are the "changeit" default, valid for 9000 days): + * + * keytool -genkeypair -alias localhost -keyalg RSA -keysize 2048 -validity 9000 \ + * -dname "CN=localhost" -ext "SAN=dns:localhost,ip:127.0.0.1" \ + * -keystore localhost.p12 -storetype PKCS12 -storepass changeit -keypass changeit + * + * keytool -genkeypair -alias otherhost -keyalg RSA -keysize 2048 -validity 9000 \ + * -dname "CN=otherhost.invalid" -ext "SAN=dns:otherhost.invalid" \ + * -keystore otherhost.p12 -storetype PKCS12 -storepass changeit -keypass changeit + */ class OkHttpTrustEverythingTest { private static final String KEYSTORE_PASSWORD = "changeit"; @@ -204,6 +216,54 @@ void cookiesAreSentWhenExplicitlyAllowed() throws Exception { getRequestedFor(urlPathEqualTo("/cookies")).withHeader("Cookie", equalTo("sid=x"))); } + @Test + void basicAuthIsWithheldOverCleartextHttp() throws Exception { + // a cleartext http:// request does not authenticate the server either + final Config conf = config(); + conf.put("http.basicauth.user", "user"); + conf.put("http.basicauth.password", "secret"); + startServer(LOCALHOST_KEYSTORE); + final ProtocolResponse response = + fetchUrl(protocol(conf), httpUrl("/cleartext"), new Metadata()); + assertEquals(200, response.getStatusCode(), "the connection must succeed"); + server.verify( + 1, getRequestedFor(urlPathEqualTo("/cleartext")).withoutHeader("Authorization")); + } + + @Test + void basicAuthIsSentOverCleartextHttpWhenExplicitlyAllowed() throws Exception { + final Config conf = config(); + conf.put("http.credentials.allow.insecure", true); + conf.put("http.basicauth.user", "user"); + conf.put("http.basicauth.password", "secret"); + startServer(LOCALHOST_KEYSTORE); + fetchUrl(protocol(conf), httpUrl("/cleartext"), new Metadata()); + final String expected = "Basic " + base64("user:secret"); + server.verify( + 1, + getRequestedFor(urlPathEqualTo("/cleartext")) + .withHeader("Authorization", equalTo(expected))); + } + + @Test + void credentialHeaderNamesCanBeConfigured() throws Exception { + // http.credentials.headers replaces the default list + final Config conf = config(); + conf.put("http.trust.everything", true); + conf.put("http.credentials.headers", List.of("X-Auth-Token")); + conf.put("http.custom.headers", List.of("X-Auth-Token=token1", "X-Api-Key=key1")); + startServer(LOCALHOST_KEYSTORE); + fetch(protocol(conf), "/configuredheaders"); + server.verify( + 1, + getRequestedFor(urlPathEqualTo("/configuredheaders")) + .withoutHeader("X-Auth-Token")); + server.verify( + 1, + getRequestedFor(urlPathEqualTo("/configuredheaders")) + .withHeader("X-Api-Key", equalTo("key1"))); + } + /** Metadata as an outlink would inherit it, with a cookie scoped to the server. */ private Metadata metadata() { final Metadata md = new Metadata(); @@ -212,6 +272,14 @@ private Metadata metadata() { return md; } + private String base64(String value) { + return Base64.getEncoder().encodeToString(value.getBytes(StandardCharsets.UTF_8)); + } + + private String httpUrl(String path) { + return "http://localhost:" + server.port() + path; + } + private Config config() { final Config conf = new Config(); conf.put("http.agent.name", "test"); @@ -235,7 +303,12 @@ private ProtocolResponse fetch(HttpProtocol protocol, String path) throws Except private ProtocolResponse fetch(HttpProtocol protocol, String path, Metadata md) throws Exception { - return protocol.getProtocolOutput("https://localhost:" + server.httpsPort() + path, md); + return fetchUrl(protocol, "https://localhost:" + server.httpsPort() + path, md); + } + + private ProtocolResponse fetchUrl(HttpProtocol protocol, String url, Metadata md) + throws Exception { + return protocol.getProtocolOutput(url, md); } /** diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc index 75e77668b..1e4cd8a19 100644 --- a/docs/src/main/asciidoc/configuration.adoc +++ b/docs/src/main/asciidoc/configuration.adoc @@ -261,9 +261,10 @@ header. | http.content.partial.as.trimmed | false | Accepts partially fetched content in OKHTTP. | http.filter.ipaddress.include | - | (OkHttp only) Comma-separated list (or YAML list) of allowed IP ranges. If empty, all addresses are allowed unless excluded. See <>. | http.filter.ipaddress.exclude | - | (OkHttp only) Comma-separated list (or YAML list) of blocked IP ranges. See <>. -| http.trust.everything | false | (OkHttp only) If true, accept any TLS certificate, including self-signed, expired or otherwise invalid ones. The servers are then not authenticated: anyone able to answer for the host name receives everything sent to them. Credentials (basic auth, credential headers, cookies) are withheld over such connections unless `http.credentials.allow.insecure` is set to true. +| http.trust.everything | false | (OkHttp only) If true, accept any TLS certificate, including self-signed, expired or otherwise invalid ones. The servers are then not authenticated: anyone able to answer for the host name receives everything sent to them. Credentials (basic auth, credential headers, cookies) are withheld from cleartext http:// requests and from https:// requests on such connections unless `http.credentials.allow.insecure` is set to true. | http.verify.hostnames | true | (OkHttp only) If true, check that the certificate presented by the server matches the host name contacted. Independent of `http.trust.everything`: a valid certificate for a different name is accepted when this is false. -| http.credentials.allow.insecure | false | (OkHttp only) If true, send credentials (basic auth, credential headers, cookies) even over connections whose server certificate was not validated, i.e. when `http.trust.everything` is true. +| http.credentials.allow.insecure | false | (OkHttp only) If true, send credentials (basic auth, credential headers, cookies) even on connections which do not authenticate the server: cleartext http:// urls and, when `http.trust.everything` is true, https:// urls as well. +| http.credentials.headers | authorization, proxy-authorization, cookie, x-api-key | (OkHttp only) Header names (case-insensitive) which carry credentials and are therefore withheld from requests to servers which were not authenticated. Applies to the headers configured with `http.custom.headers` and to headers set by request. When set, this key replaces the default list. | topology.message.timeout.secs | -1 | OKHTTP message timeout. |=== diff --git a/docs/src/main/asciidoc/extending.adoc b/docs/src/main/asciidoc/extending.adoc index a22494295..8b1e1d6fd 100644 --- a/docs/src/main/asciidoc/extending.adoc +++ b/docs/src/main/asciidoc/extending.adoc @@ -392,7 +392,7 @@ Keep the pool size (idle + active) under 1000. For large crawls, increase `proto ==== SSL/TLS -By default, StormCrawler's OkHttp protocol validates the certificate chains of HTTPS servers and checks that the certificate matches the host name contacted (`http.trust.everything: false`, `http.verify.hostnames: true`). For crawls of hosts whose certificates cannot be validated, e.g. self-signed ones, set `http.trust.everything` to `true`: the servers are then not authenticated, so credentials (basic auth, credential headers in `http.custom.headers`, cookies) are withheld over such connections unless `http.credentials.allow.insecure` is set to `true`. Hostname checking is a separate decision and can be disabled with `http.verify.hostnames: false`. +By default, StormCrawler's OkHttp protocol validates the certificate chains of HTTPS servers and checks that the certificate matches the host name contacted (`http.trust.everything: false`, `http.verify.hostnames: true`). For crawls of hosts whose certificates cannot be validated, e.g. self-signed ones, set `http.trust.everything` to `true`: the servers are then not authenticated, so credentials (basic auth, credential headers in `http.custom.headers`, cookies) are withheld unless `http.credentials.allow.insecure` is set to `true`. Credentials are also withheld from cleartext http:// requests, which never authenticate the server. Which header names count as credentials is configured with `http.credentials.headers`. Hostname checking is a separate decision and can be disabled with `http.verify.hostnames: false`. ==== Authentication @@ -404,7 +404,7 @@ http.basicauth.user: "username" http.basicauth.password: "password" ---- -These credentials are sent as an `Authorization` header with every request to servers authenticated by their TLS certificate; they are withheld when `http.trust.everything` is enabled, unless `http.credentials.allow.insecure` is set to `true`. For per-site authentication, use metadata-driven headers instead (see <> in the Internals section). +These credentials are sent as an `Authorization` header with requests to servers authenticated by their TLS certificate; they are withheld from cleartext http:// requests and from https:// requests when `http.trust.everything` is enabled, unless `http.credentials.allow.insecure` is set to `true`. For per-site authentication, use metadata-driven headers instead (see <> in the Internals section). ==== Proxy Authentication From 9a7a1281d01f3e9ca04753b6ea26276ad40d913d Mon Sep 17 00:00:00 2001 From: abhinav-phi Date: Wed, 9 Sep 2026 23:29:28 +0530 Subject: [PATCH 4/7] Enforce credential policy on redirect hops and hostname verification (#2079) - credentialsAllowed now requires http.verify.hostnames: a trusted cert for the wrong host with verification off no longer receives credentials - network interceptor strips credential headers on every hop whose URL is not authenticated, covering automatic HTTPS->HTTP redirects - regression tests for hostname-off, per-request headers, decision matrix, and HTTPS->HTTP redirect stripping/forwarding --- .../protocol/okhttp/HttpProtocol.java | 51 ++++- .../okhttp/OkHttpTrustEverythingTest.java | 206 ++++++++++++++++++ 2 files changed, 250 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java index 17194fdd1..e9a70bb63 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java @@ -135,6 +135,11 @@ public class HttpProtocol extends AbstractHttpProtocol { // do not authenticate the servers then either private boolean trustEverything = false; + // check that the certificate matches the host name (http.verify.hostnames): + // with it off, a valid certificate for a different name is accepted and the + // server is not authenticated, see #credentialsAllowed + private boolean verifyHostnames = true; + // send credentials even to servers which were not authenticated // (http.credentials.allow.insecure) private boolean insecureCredentialsAllowed = false; @@ -195,7 +200,7 @@ public void configure(Config conf) { * accepting any certificate does not imply accepting any name */ this.trustEverything = ConfUtils.getBoolean(conf, "http.trust.everything", false); - final boolean verifyHostnames = ConfUtils.getBoolean(conf, "http.verify.hostnames", true); + this.verifyHostnames = ConfUtils.getBoolean(conf, "http.verify.hostnames", true); // credentials are withheld on connections which do not authenticate the // server, unless explicitly opted in this.insecureCredentialsAllowed = @@ -340,6 +345,33 @@ public void configure(Config conf) { builder.addNetworkInterceptor(new HTTPHeadersInterceptor()); } + // Enforce the credential policy on every network hop, including the + // automatic redirects OkHttp follows when http.allow.redirects is true: + // the initial request is already filtered in getProtocolOutput, but the + // follower copies headers like Authorization or X-Api-Key onto the next + // request without re-checking. A network interceptor sees each hop, so + // credentials never reach a server which is not authenticated, e.g. a + // cleartext http:// redirect target of a trusted https:// URL. + // Proxy-Authorization is left alone: it authenticates against the proxy + // itself, not the crawled server, like the proxyAuthenticator below. + builder.addNetworkInterceptor( + chain -> { + Request hop = chain.request(); + if (!credentialsAllowed(hop.url().toString())) { + Request.Builder stripped = hop.newBuilder(); + for (String name : new HashSet<>(hop.headers().names())) { + if (HttpHeaders.PROXY_AUTHORIZATION.equalsIgnoreCase(name)) { + continue; + } + if (isCredentialHeader(name)) { + stripped.removeHeader(name); + } + } + hop = stripped.build(); + } + return chain.proceed(hop); + }); + if (trustEverything) { builder.sslSocketFactory(trustAllSslSocketFactory, (X509TrustManager) trustAllCerts[0]); } @@ -451,17 +483,22 @@ private boolean isCredentialHeader(String name) { /** * Returns true when credentials (basic auth, credential headers, cookies) may be sent with the - * request to the url. A server is only authenticated when the connection is HTTPS and the - * certificate chain is validated; on cleartext http:// connections and on https:// connections - * which accept any certificate (http.trust.everything), credentials are withheld unless + * request to the url. A server is only authenticated when the connection is HTTPS, the + * certificate chain is validated and the certificate is checked against the host name + * contacted; on cleartext http:// connections, on https:// connections which accept any + * certificate (http.trust.everything) and on https:// connections which skip the host name + * check (http.verify.hostnames), credentials are withheld unless * http.credentials.allow.insecure is enabled. */ - private boolean credentialsAllowed(String url) { + // package-private for the decision matrix in OkHttpTrustEverythingTest + boolean credentialsAllowed(String url) { if (insecureCredentialsAllowed) { return true; } try { - return URLUtil.toURL(url).getProtocol().equals("https") && !trustEverything; + return URLUtil.toURL(url).getProtocol().equals("https") + && !trustEverything + && verifyHostnames; } catch (MalformedURLException e) { return false; } @@ -571,7 +608,7 @@ public ProtocolResponse getProtocolOutput(String url, final Metadata metadata) + "withheld because {} is not authenticated. Set " + "http.credentials.allow.insecure to true to send them anyway.", url.startsWith("https") - ? "https with http.trust.everything" + ? "https with http.trust.everything or without http.verify.hostnames" : "cleartext http"); } diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java index a0ff5707b..65fe3aacc 100644 --- a/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java +++ b/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java @@ -17,27 +17,39 @@ package org.apache.stormcrawler.protocol.okhttp; +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; import static com.github.tomakehurst.wiremock.client.WireMock.any; import static com.github.tomakehurst.wiremock.client.WireMock.anyUrl; import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; +import static com.github.tomakehurst.wiremock.client.WireMock.get; import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor; import static com.github.tomakehurst.wiremock.client.WireMock.ok; import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.github.tomakehurst.wiremock.WireMockServer; import com.github.tomakehurst.wiremock.common.ConsoleNotifier; import com.github.tomakehurst.wiremock.core.WireMockConfiguration; import java.io.InputStream; +import java.lang.reflect.Field; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; +import java.security.KeyStore; +import java.security.SecureRandom; import java.util.Base64; import java.util.List; +import javax.net.ssl.SSLContext; import javax.net.ssl.SSLHandshakeException; import javax.net.ssl.SSLPeerUnverifiedException; +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; +import javax.net.ssl.X509TrustManager; +import okhttp3.OkHttpClient; import org.apache.storm.Config; import org.apache.stormcrawler.Metadata; import org.apache.stormcrawler.protocol.ProtocolResponse; @@ -264,6 +276,200 @@ void credentialHeaderNamesCanBeConfigured() throws Exception { .withHeader("X-Api-Key", equalTo("key1"))); } + /** + * Credentials must also be withheld when hostname verification is switched off: a valid + * certificate for a different name, accepted only because the name is not checked, does not + * authenticate the server any more than a trust-all chain does. + */ + @Test + void credentialsAreWithheldWhenHostnameVerificationIsDisabled() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + conf.put("http.verify.hostnames", false); + conf.put("http.basicauth.user", "user"); + conf.put("http.basicauth.password", "secret"); + startServer(OTHERHOST_KEYSTORE); + final ProtocolResponse response = fetch(protocol(conf), "/nohostnamecheck"); + assertEquals(200, response.getStatusCode(), "the connection succeeds as configured"); + server.verify( + 1, + getRequestedFor(urlPathEqualTo("/nohostnamecheck")).withoutHeader("Authorization")); + } + + /** The opt-in still sends them on that path. */ + @Test + void credentialsAreSentWhenHostnameVerificationIsDisabledAndInsecureAllowed() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + conf.put("http.verify.hostnames", false); + conf.put("http.credentials.allow.insecure", true); + conf.put("http.basicauth.user", "user"); + conf.put("http.basicauth.password", "secret"); + startServer(OTHERHOST_KEYSTORE); + fetch(protocol(conf), "/nohostnamecheckinsecure"); + final String expected = "Basic " + base64("user:secret"); + server.verify( + 1, + getRequestedFor(urlPathEqualTo("/nohostnamecheckinsecure")) + .withHeader("Authorization", equalTo(expected))); + } + + /** Credential headers set per request through metadata go through the same policy. */ + @Test + void credentialHeaderSetByRequestIsWithheldFromUnauthenticatedServers() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + startServer(LOCALHOST_KEYSTORE); + final Metadata md = new Metadata(); + md.setValue("protocol.set-header", "X-Api-Key=s3cret"); + md.setValue("protocol.set-header", "X-Trace=public"); + fetch(protocol(conf), "/setheaders", md); + server.verify(1, getRequestedFor(urlPathEqualTo("/setheaders")).withoutHeader("X-Api-Key")); + server.verify( + 1, + getRequestedFor(urlPathEqualTo("/setheaders")) + .withHeader("X-Trace", equalTo("public"))); + } + + @Test + void credentialHeaderSetByRequestIsSentWhenExplicitlyAllowed() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + conf.put("http.credentials.allow.insecure", true); + startServer(LOCALHOST_KEYSTORE); + final Metadata md = new Metadata(); + md.setValue("protocol.set-header", "X-Api-Key=s3cret"); + fetch(protocol(conf), "/setheadersallowed", md); + server.verify( + 1, + getRequestedFor(urlPathEqualTo("/setheadersallowed")) + .withHeader("X-Api-Key", equalTo("s3cret"))); + } + + /** + * The decision itself over the full configuration matrix, including the positive case of a + * normally validated HTTPS connection, which the WireMock tests above cannot express because + * their certificates are self-signed. + */ + @Test + void credentialsAllowedDecision() { + // default: chains validated, hostnames checked -> https is authenticated + HttpProtocol p = protocol(config()); + assertTrue(p.credentialsAllowed("https://example.org/"), "validated https sends"); + assertFalse(p.credentialsAllowed("http://example.org/"), "cleartext never sends"); + // trust-all: chains are not validated + final Config trustAll = config(); + trustAll.put("http.trust.everything", true); + assertFalse(protocol(trustAll).credentialsAllowed("https://example.org/")); + // hostname verification disabled: a name mismatch is not caught + final Config noHostCheck = config(); + noHostCheck.put("http.verify.hostnames", false); + assertFalse(protocol(noHostCheck).credentialsAllowed("https://example.org/")); + // the opt-in covers every case + final Config allowInsecure = config(); + allowInsecure.put("http.credentials.allow.insecure", true); + assertTrue(protocol(allowInsecure).credentialsAllowed("http://example.org/")); + } + + /** + * An automatic redirect from trusted HTTPS to cleartext HTTP must not forward credentials: the + * policy is enforced on every hop, not just the initial URL. The self-signed test certificate + * is trusted via the client transport (without flipping http.trust.everything, which would also + * flip the policy), so the initial HTTPS hop is authenticated and sends credentials while the + * HTTP hop must not receive them. + */ + @Test + void credentialsAreStrippedOnHttpsToHttpRedirect() throws Exception { + final Config conf = config(); + conf.put("http.allow.redirects", true); + conf.put("http.basicauth.user", "user"); + conf.put("http.basicauth.password", "secret"); + conf.put("http.custom.headers", List.of("X-Api-Key=s3cret")); + startServer(LOCALHOST_KEYSTORE); + final HttpProtocol protocol = protocol(conf); + trustTestKeystore(protocol, LOCALHOST_KEYSTORE); + server.stubFor( + get(urlPathEqualTo("/redirect")) + .atPriority(1) + .willReturn( + aResponse() + .withStatus(302) + .withHeader("Location", httpUrl("/target")))); + final ProtocolResponse response = + fetchUrl( + protocol, + "https://localhost:" + server.httpsPort() + "/redirect", + new Metadata()); + assertEquals(200, response.getStatusCode(), "the redirect is followed"); + final String expected = "Basic " + base64("user:secret"); + server.verify( + 1, + getRequestedFor(urlPathEqualTo("/redirect")) + .withHeader("Authorization", equalTo(expected))); + server.verify(1, getRequestedFor(urlPathEqualTo("/target")).withoutHeader("Authorization")); + server.verify(1, getRequestedFor(urlPathEqualTo("/target")).withoutHeader("X-Api-Key")); + } + + /** + * With the opt-in, the same redirect forwards custom credential headers to the HTTP hop. + * (OkHttp itself strips Authorization on a scheme downgrade even then, which is its safe + * default; the policy opt-in covers the headers StormCrawler controls.) + */ + @Test + void credentialsAreForwardedOnHttpsToHttpRedirectWhenInsecureAllowed() throws Exception { + final Config conf = config(); + conf.put("http.allow.redirects", true); + conf.put("http.credentials.allow.insecure", true); + conf.put("http.basicauth.user", "user"); + conf.put("http.basicauth.password", "secret"); + conf.put("http.custom.headers", List.of("X-Api-Key=s3cret")); + startServer(LOCALHOST_KEYSTORE); + final HttpProtocol protocol = protocol(conf); + trustTestKeystore(protocol, LOCALHOST_KEYSTORE); + server.stubFor( + get(urlPathEqualTo("/redirect-allowed")) + .atPriority(1) + .willReturn( + aResponse() + .withStatus(302) + .withHeader("Location", httpUrl("/target-allowed")))); + fetchUrl( + protocol, + "https://localhost:" + server.httpsPort() + "/redirect-allowed", + new Metadata()); + server.verify( + 1, + getRequestedFor(urlPathEqualTo("/target-allowed")) + .withHeader("X-Api-Key", equalTo("s3cret"))); + } + + /** + * Trusts the self-signed test keystore on the protocol transport without changing the + * credential policy: http.trust.everything stays false, so credentialsAllowed still treats the + * HTTPS URL as authenticated, while the TLS handshake succeeds. + */ + private void trustTestKeystore(HttpProtocol protocol, String keystoreResource) + throws Exception { + final KeyStore trustStore = KeyStore.getInstance("PKCS12"); + try (InputStream in = getClass().getResourceAsStream(keystoreResource)) { + trustStore.load(in, KEYSTORE_PASSWORD.toCharArray()); + } + final TrustManagerFactory tmf = + TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + tmf.init(trustStore); + final X509TrustManager trustManager = (X509TrustManager) tmf.getTrustManagers()[0]; + final SSLContext sslContext = SSLContext.getInstance("TLS"); + sslContext.init(null, new TrustManager[] {trustManager}, new SecureRandom()); + final Field clientField = HttpProtocol.class.getDeclaredField("client"); + clientField.setAccessible(true); + final OkHttpClient client = (OkHttpClient) clientField.get(protocol); + final OkHttpClient trusted = + client.newBuilder() + .sslSocketFactory(sslContext.getSocketFactory(), trustManager) + .build(); + clientField.set(protocol, trusted); + } + /** Metadata as an outlink would inherit it, with a cookie scoped to the server. */ private Metadata metadata() { final Metadata md = new Metadata(); From 022356d5969dec2d8e30ea0d4e7598e2b987776e Mon Sep 17 00:00:00 2001 From: Richard Zowalla Date: Wed, 9 Sep 2026 21:25:45 +0200 Subject: [PATCH 5/7] http.credentials.headers adds to the built-in names Setting the key replaced the whole list, so the example in crawler-default.yaml ("x-api-key", "x-auth-token") dropped authorization, proxy-authorization and cookie from it. A header configured through http.custom.headers under one of those names was then classified as an ordinary header and sent to servers which were not authenticated, and the redirect interceptor stopped stripping it on the next hop. The configured names are merged into the defaults instead, and the yaml and configuration.adoc say so. Also decide the policy on the HttpUrl the interceptor already holds rather than re-parsing the string, and trim the comments added by this branch. --- .../protocol/okhttp/HttpProtocol.java | 82 ++++++------------- core/src/main/resources/crawler-default.yaml | 11 ++- .../okhttp/OkHttpTrustEverythingTest.java | 24 +++++- docs/src/main/asciidoc/configuration.adoc | 2 +- 4 files changed, 52 insertions(+), 67 deletions(-) diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java index e9a70bb63..dff27216e 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java @@ -54,6 +54,7 @@ import okhttp3.Gzip; import okhttp3.Handshake; import okhttp3.Headers; +import okhttp3.HttpUrl; import okhttp3.Interceptor; import okhttp3.MediaType; import okhttp3.OkHttpClient; @@ -123,25 +124,19 @@ public class HttpProtocol extends AbstractHttpProtocol { "cookie", "x-api-key"); - // header names (lower case) considered to carry credentials, configured with - // http.credentials.headers, see #isCredentialHeader + // lower case header names considered to carry credentials private Set credentialHeaders = DEFAULT_CREDENTIAL_HEADERS; - // request headers which carry credentials; they are only sent to servers - // which were authenticated, see #credentialsAllowed + // request headers withheld from servers which were not authenticated private final List credentialRequestHeaders = new LinkedList<>(); - // accept any certificate chain (http.trust.everything): https connections - // do not authenticate the servers then either + // http.trust.everything: accept any certificate chain private boolean trustEverything = false; - // check that the certificate matches the host name (http.verify.hostnames): - // with it off, a valid certificate for a different name is accepted and the - // server is not authenticated, see #credentialsAllowed + // http.verify.hostnames: check the certificate against the host name contacted private boolean verifyHostnames = true; - // send credentials even to servers which were not authenticated - // (http.credentials.allow.insecure) + // http.credentials.allow.insecure private boolean insecureCredentialsAllowed = false; private OkHttpClient.Builder builder; @@ -195,23 +190,16 @@ public void configure(Config conf) { this.partialContentAsTrimmed = ConfUtils.getBoolean(conf, "http.content.partial.as.trimmed", false); - /* - * certificate trust and hostname verification are separate decisions: - * accepting any certificate does not imply accepting any name - */ this.trustEverything = ConfUtils.getBoolean(conf, "http.trust.everything", false); this.verifyHostnames = ConfUtils.getBoolean(conf, "http.verify.hostnames", true); - // credentials are withheld on connections which do not authenticate the - // server, unless explicitly opted in this.insecureCredentialsAllowed = ConfUtils.getBoolean(conf, "http.credentials.allow.insecure", false); - // header names considered to carry credentials; when set, - // http.credentials.headers replaces the default list + // http.credentials.headers adds names to the built-in ones, it cannot remove them final List configuredCredentialHeaders = ConfUtils.loadListFromConf("http.credentials.headers", conf); if (!configuredCredentialHeaders.isEmpty()) { - final Set names = new HashSet<>(); + final Set names = new HashSet<>(DEFAULT_CREDENTIAL_HEADERS); for (String name : configuredCredentialHeaders) { if (StringUtils.isNotBlank(name)) { names.add(name.trim().toLowerCase(Locale.ROOT)); @@ -294,8 +282,7 @@ public void configure(Config conf) { final String basicAuthUser = ConfUtils.getString(conf, "http.basicauth.user", null); - // use a basic auth? the Authorization header carries credentials and is - // only sent to servers which were authenticated, see #credentialsAllowed + // use a basic auth? the header is withheld unless the server was authenticated if (StringUtils.isNotBlank(basicAuthUser)) { final String basicAuthPass = ConfUtils.getString(conf, "http.basicauth.password", ""); final String encoding = @@ -345,19 +332,13 @@ public void configure(Config conf) { builder.addNetworkInterceptor(new HTTPHeadersInterceptor()); } - // Enforce the credential policy on every network hop, including the - // automatic redirects OkHttp follows when http.allow.redirects is true: - // the initial request is already filtered in getProtocolOutput, but the - // follower copies headers like Authorization or X-Api-Key onto the next - // request without re-checking. A network interceptor sees each hop, so - // credentials never reach a server which is not authenticated, e.g. a - // cleartext http:// redirect target of a trusted https:// URL. - // Proxy-Authorization is left alone: it authenticates against the proxy - // itself, not the crawled server, like the proxyAuthenticator below. + // getProtocolOutput only filters the initial request, the redirect follower copies + // the headers onto the next hop without re-checking. Proxy-Authorization is left + // alone, it authenticates against the proxy rather than the crawled server. builder.addNetworkInterceptor( chain -> { Request hop = chain.request(); - if (!credentialsAllowed(hop.url().toString())) { + if (!credentialsAllowed(hop.url())) { Request.Builder stripped = hop.newBuilder(); for (String name : new HashSet<>(hop.headers().names())) { if (HttpHeaders.PROXY_AUTHORIZATION.equalsIgnoreCase(name)) { @@ -468,11 +449,7 @@ private URL getCookieOrigin(Metadata md, String url) { } } - /** - * Returns true when the header carries credentials which must not be disclosed to servers that - * were not authenticated. The names are configured with http.credentials.headers and default to - * the standard credential headers plus any header commonly used for API keys. - */ + /** Whether the header carries credentials, see http.credentials.headers. */ private boolean isCredentialHeader(String name) { if (name == null) { return false; @@ -482,26 +459,21 @@ private boolean isCredentialHeader(String name) { } /** - * Returns true when credentials (basic auth, credential headers, cookies) may be sent with the - * request to the url. A server is only authenticated when the connection is HTTPS, the - * certificate chain is validated and the certificate is checked against the host name - * contacted; on cleartext http:// connections, on https:// connections which accept any - * certificate (http.trust.everything) and on https:// connections which skip the host name - * check (http.verify.hostnames), credentials are withheld unless - * http.credentials.allow.insecure is enabled. + * Whether credentials may be sent to the url. The server is only authenticated over https with + * both the certificate chain and the host name checked; http.credentials.allow.insecure sends + * them anyway. */ // package-private for the decision matrix in OkHttpTrustEverythingTest boolean credentialsAllowed(String url) { if (insecureCredentialsAllowed) { return true; } - try { - return URLUtil.toURL(url).getProtocol().equals("https") - && !trustEverything - && verifyHostnames; - } catch (MalformedURLException e) { - return false; - } + final HttpUrl parsed = HttpUrl.parse(url); + return parsed != null && credentialsAllowed(parsed); + } + + private boolean credentialsAllowed(HttpUrl url) { + return insecureCredentialsAllowed || (url.isHttps() && !trustEverything && verifyHostnames); } protected void addHeadersToRequest(Builder rb, Metadata md, boolean sendCredentials) { @@ -556,10 +528,8 @@ public ProtocolResponse getProtocolOutput(String url, final Metadata metadata) // conditionally add proxy authentication if (StringUtils.isNotBlank(prox.getUsername())) { - // the proxy is not the crawled server: its credentials - // authenticate against the proxy itself and are sent - // regardless of whether the target server was - // authenticated, cf. #credentialsAllowed + // authenticates against the proxy, not the crawled server, + // so it is sent whatever credentialsAllowed decides localBuilder.proxyAuthenticator( (Route route, Response response) -> { String credential = @@ -587,8 +557,6 @@ public ProtocolResponse getProtocolOutput(String url, final Metadata metadata) } } - // credentials are only sent to servers which were authenticated, which - // depends on the url of the individual request final boolean sendCredentials = credentialsAllowed(url); final Builder rb = new Request.Builder().url(url); diff --git a/core/src/main/resources/crawler-default.yaml b/core/src/main/resources/crawler-default.yaml index 3e2a4b170..80cd58ad7 100644 --- a/core/src/main/resources/crawler-default.yaml +++ b/core/src/main/resources/crawler-default.yaml @@ -175,15 +175,14 @@ config: # were never authenticated. http.credentials.allow.insecure: false - # Header names (case-insensitive) which carry credentials and are therefore - # withheld from requests to servers which were not authenticated (okhttp - # protocol only, see http.trust.everything and + # Additional header names (case-insensitive) which carry credentials and are + # therefore withheld from requests to servers which were not authenticated + # (okhttp protocol only, see http.trust.everything and # http.credentials.allow.insecure). Applies to the headers configured with # http.custom.headers and to headers set by request; cookies are withheld - # separately. When set, this key replaces the default list of - # "authorization", "proxy-authorization", "cookie" and "x-api-key". + # separately. "authorization", "proxy-authorization", "cookie" and + # "x-api-key" are always treated as credentials and cannot be removed here. # http.credentials.headers: - # - "x-api-key" # - "x-auth-token" # IP address filtering (okhttp protocol only). Optionally limit or block diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java index 65fe3aacc..f909312d9 100644 --- a/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java +++ b/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java @@ -259,11 +259,10 @@ void basicAuthIsSentOverCleartextHttpWhenExplicitlyAllowed() throws Exception { @Test void credentialHeaderNamesCanBeConfigured() throws Exception { - // http.credentials.headers replaces the default list final Config conf = config(); conf.put("http.trust.everything", true); conf.put("http.credentials.headers", List.of("X-Auth-Token")); - conf.put("http.custom.headers", List.of("X-Auth-Token=token1", "X-Api-Key=key1")); + conf.put("http.custom.headers", List.of("X-Auth-Token=token1", "X-Other=plain")); startServer(LOCALHOST_KEYSTORE); fetch(protocol(conf), "/configuredheaders"); server.verify( @@ -273,7 +272,26 @@ void credentialHeaderNamesCanBeConfigured() throws Exception { server.verify( 1, getRequestedFor(urlPathEqualTo("/configuredheaders")) - .withHeader("X-Api-Key", equalTo("key1"))); + .withHeader("X-Other", equalTo("plain"))); + } + + /** http.credentials.headers adds to the built-in names, it does not replace them. */ + @Test + void configuredHeaderNamesDoNotDropTheBuiltInOnes() throws Exception { + final Config conf = config(); + conf.put("http.trust.everything", true); + conf.put("http.credentials.headers", List.of("x-auth-token")); + conf.put( + "http.custom.headers", + List.of("Authorization=Basic c2VjcmV0", "X-Api-Key=key1", "Cookie=sid=x")); + startServer(LOCALHOST_KEYSTORE); + fetch(protocol(conf), "/builtinheaders"); + server.verify( + 1, + getRequestedFor(urlPathEqualTo("/builtinheaders")) + .withoutHeader("Authorization") + .withoutHeader("X-Api-Key") + .withoutHeader("Cookie")); } /** diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc index 1e4cd8a19..5b66d6f05 100644 --- a/docs/src/main/asciidoc/configuration.adoc +++ b/docs/src/main/asciidoc/configuration.adoc @@ -264,7 +264,7 @@ header. | http.trust.everything | false | (OkHttp only) If true, accept any TLS certificate, including self-signed, expired or otherwise invalid ones. The servers are then not authenticated: anyone able to answer for the host name receives everything sent to them. Credentials (basic auth, credential headers, cookies) are withheld from cleartext http:// requests and from https:// requests on such connections unless `http.credentials.allow.insecure` is set to true. | http.verify.hostnames | true | (OkHttp only) If true, check that the certificate presented by the server matches the host name contacted. Independent of `http.trust.everything`: a valid certificate for a different name is accepted when this is false. | http.credentials.allow.insecure | false | (OkHttp only) If true, send credentials (basic auth, credential headers, cookies) even on connections which do not authenticate the server: cleartext http:// urls and, when `http.trust.everything` is true, https:// urls as well. -| http.credentials.headers | authorization, proxy-authorization, cookie, x-api-key | (OkHttp only) Header names (case-insensitive) which carry credentials and are therefore withheld from requests to servers which were not authenticated. Applies to the headers configured with `http.custom.headers` and to headers set by request. When set, this key replaces the default list. +| http.credentials.headers | authorization, proxy-authorization, cookie, x-api-key | (OkHttp only) Header names (case-insensitive) which carry credentials and are therefore withheld from requests to servers which were not authenticated. Applies to the headers configured with `http.custom.headers` and to headers set by request. Names given here are added to the four defaults, which are always treated as credentials. | topology.message.timeout.secs | -1 | OKHTTP message timeout. |=== From 18c696c6f5b75f060c9ee370e981c7cab11efa5e Mon Sep 17 00:00:00 2001 From: Richard Zowalla Date: Thu, 10 Sep 2026 09:56:06 +0200 Subject: [PATCH 6/7] Strip credentials before recording headers, withhold static Proxy-Authorization on hops The credential interceptor now runs before HTTPHeadersInterceptor, so the request headers stored for a hop (and written to WARC request records) are the ones actually sent, not the ones stripped from a cleartext hop. Proxy-Authorization is only kept on a hop when the proxy authenticator set it and the hop is cleartext, i.e. when the proxy reads it. A static Proxy-Authorization from http.custom.headers is treated like any other credential on every hop, as the configuration documents. Document that http.verify.hostnames: false withholds credentials too. --- .../protocol/okhttp/HttpProtocol.java | 26 +++++-- core/src/main/resources/crawler-default.yaml | 9 ++- .../okhttp/OkHttpTrustEverythingTest.java | 68 +++++++++++++++++++ docs/src/main/asciidoc/configuration.adoc | 4 +- docs/src/main/asciidoc/extending.adoc | 4 +- 5 files changed, 97 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java index dff27216e..5597d9f87 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java @@ -139,6 +139,11 @@ public class HttpProtocol extends AbstractHttpProtocol { // http.credentials.allow.insecure private boolean insecureCredentialsAllowed = false; + /** Tags a request whose Proxy-Authorization was set by the proxy authenticator. */ + private enum ProxyAuthenticated { + INSTANCE + } + private OkHttpClient.Builder builder; private static final TrustManager[] trustAllCerts = @@ -328,20 +333,20 @@ public void configure(Config conf) { builder.addNetworkInterceptor(new HTTPFilterIPAddressInterceptor(ipFilterRules)); } - if (storeHttpHeaders) { - builder.addNetworkInterceptor(new HTTPHeadersInterceptor()); - } - // getProtocolOutput only filters the initial request, the redirect follower copies - // the headers onto the next hop without re-checking. Proxy-Authorization is left - // alone, it authenticates against the proxy rather than the crawled server. + // the headers onto the next hop without re-checking. Registered before the + // HTTPHeadersInterceptor so that the request headers it records are the ones sent. builder.addNetworkInterceptor( chain -> { Request hop = chain.request(); if (!credentialsAllowed(hop.url())) { + // the proxy authenticator's header is read by the proxy on a cleartext + // hop, anywhere else Proxy-Authorization reaches the crawled server + final boolean forProxy = + hop.tag(ProxyAuthenticated.class) != null && !hop.url().isHttps(); Request.Builder stripped = hop.newBuilder(); for (String name : new HashSet<>(hop.headers().names())) { - if (HttpHeaders.PROXY_AUTHORIZATION.equalsIgnoreCase(name)) { + if (forProxy && HttpHeaders.PROXY_AUTHORIZATION.equalsIgnoreCase(name)) { continue; } if (isCredentialHeader(name)) { @@ -353,6 +358,10 @@ public void configure(Config conf) { return chain.proceed(hop); }); + if (storeHttpHeaders) { + builder.addNetworkInterceptor(new HTTPHeadersInterceptor()); + } + if (trustEverything) { builder.sslSocketFactory(trustAllSslSocketFactory, (X509TrustManager) trustAllCerts[0]); } @@ -538,6 +547,9 @@ public ProtocolResponse getProtocolOutput(String url, final Metadata metadata) return response.request() .newBuilder() .header(HttpHeaders.PROXY_AUTHORIZATION, credential) + .tag( + ProxyAuthenticated.class, + ProxyAuthenticated.INSTANCE) .build(); }); } diff --git a/core/src/main/resources/crawler-default.yaml b/core/src/main/resources/crawler-default.yaml index 80cd58ad7..f20418203 100644 --- a/core/src/main/resources/crawler-default.yaml +++ b/core/src/main/resources/crawler-default.yaml @@ -164,15 +164,18 @@ config: # contacted (okhttp protocol only)? Disabling this is independent of # http.trust.everything: a valid certificate for a different name is then # accepted. Hostname verification does not need to be disabled for hosts - # with self-signed certificates when http.trust.everything is enabled. + # with self-signed certificates when http.trust.everything is enabled. When + # disabled, credentials are withheld unless http.credentials.allow.insecure + # is set to true. http.verify.hostnames: true # Send credentials (basic auth, credential headers, cookies) even on # connections which do not authenticate the server (okhttp protocol only)? # Without it, credentials are withheld from cleartext http:// requests and # from https:// requests whose certificate was not validated - # (http.trust.everything), so that they are not disclosed to servers which - # were never authenticated. + # (http.trust.everything) or not checked against the host name + # (http.verify.hostnames: false), so that they are not disclosed to servers + # which were never authenticated. http.credentials.allow.insecure: false # Additional header names (case-insensitive) which carry credentials and are diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java index f909312d9..814ec6f90 100644 --- a/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java +++ b/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java @@ -43,6 +43,7 @@ import java.security.SecureRandom; import java.util.Base64; import java.util.List; +import java.util.Locale; import javax.net.ssl.SSLContext; import javax.net.ssl.SSLHandshakeException; import javax.net.ssl.SSLPeerUnverifiedException; @@ -461,6 +462,73 @@ void credentialsAreForwardedOnHttpsToHttpRedirectWhenInsecureAllowed() throws Ex .withHeader("X-Api-Key", equalTo("s3cret"))); } + /** + * With http.store.headers, the request headers recorded for a hop are the ones sent on it: a + * credential stripped from the cleartext hop must not show up in the stored request either. + */ + @Test + void storedRequestHeadersMatchTheStrippedHop() throws Exception { + final Config conf = config(); + conf.put("http.allow.redirects", true); + conf.put("http.store.headers", true); + conf.put("http.custom.headers", List.of("X-Api-Key=s3cret")); + startServer(LOCALHOST_KEYSTORE); + final HttpProtocol protocol = protocol(conf); + trustTestKeystore(protocol, LOCALHOST_KEYSTORE); + server.stubFor( + get(urlPathEqualTo("/redirect-stored")) + .atPriority(1) + .willReturn( + aResponse() + .withStatus(302) + .withHeader("Location", httpUrl("/target-stored")))); + final ProtocolResponse response = + fetchUrl( + protocol, + "https://localhost:" + server.httpsPort() + "/redirect-stored", + new Metadata()); + final String stored = + response.getMetadata().getFirstValue(ProtocolResponse.REQUEST_HEADERS_KEY); + assertTrue(stored.contains("/target-stored "), "the last hop is recorded: " + stored); + assertFalse( + stored.toLowerCase(Locale.ROOT).contains("x-api-key"), + "the stripped header must not be recorded: " + stored); + } + + /** + * A Proxy-Authorization header configured in http.custom.headers is withheld like any other + * credential. On a direct connection it would reach the crawled server, so a redirect to + * cleartext HTTP must strip it too. + */ + @Test + void staticProxyAuthorizationIsStrippedOnHttpsToHttpRedirect() throws Exception { + final Config conf = config(); + conf.put("http.allow.redirects", true); + conf.put("http.custom.headers", List.of("Proxy-Authorization=Basic cHJveHk6c2VjcmV0")); + startServer(LOCALHOST_KEYSTORE); + final HttpProtocol protocol = protocol(conf); + trustTestKeystore(protocol, LOCALHOST_KEYSTORE); + server.stubFor( + get(urlPathEqualTo("/redirect-proxyauth")) + .atPriority(1) + .willReturn( + aResponse() + .withStatus(302) + .withHeader("Location", httpUrl("/target-proxyauth")))); + fetchUrl( + protocol, + "https://localhost:" + server.httpsPort() + "/redirect-proxyauth", + new Metadata()); + server.verify( + 1, + getRequestedFor(urlPathEqualTo("/redirect-proxyauth")) + .withHeader("Proxy-Authorization", equalTo("Basic cHJveHk6c2VjcmV0"))); + server.verify( + 1, + getRequestedFor(urlPathEqualTo("/target-proxyauth")) + .withoutHeader("Proxy-Authorization")); + } + /** * Trusts the self-signed test keystore on the protocol transport without changing the * credential policy: http.trust.everything stays false, so credentialsAllowed still treats the diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc index 5b66d6f05..781fb5670 100644 --- a/docs/src/main/asciidoc/configuration.adoc +++ b/docs/src/main/asciidoc/configuration.adoc @@ -262,8 +262,8 @@ header. | http.filter.ipaddress.include | - | (OkHttp only) Comma-separated list (or YAML list) of allowed IP ranges. If empty, all addresses are allowed unless excluded. See <>. | http.filter.ipaddress.exclude | - | (OkHttp only) Comma-separated list (or YAML list) of blocked IP ranges. See <>. | http.trust.everything | false | (OkHttp only) If true, accept any TLS certificate, including self-signed, expired or otherwise invalid ones. The servers are then not authenticated: anyone able to answer for the host name receives everything sent to them. Credentials (basic auth, credential headers, cookies) are withheld from cleartext http:// requests and from https:// requests on such connections unless `http.credentials.allow.insecure` is set to true. -| http.verify.hostnames | true | (OkHttp only) If true, check that the certificate presented by the server matches the host name contacted. Independent of `http.trust.everything`: a valid certificate for a different name is accepted when this is false. -| http.credentials.allow.insecure | false | (OkHttp only) If true, send credentials (basic auth, credential headers, cookies) even on connections which do not authenticate the server: cleartext http:// urls and, when `http.trust.everything` is true, https:// urls as well. +| http.verify.hostnames | true | (OkHttp only) If true, check that the certificate presented by the server matches the host name contacted. Independent of `http.trust.everything`: a valid certificate for a different name is accepted when this is false. Credentials are then withheld unless `http.credentials.allow.insecure` is set to true. +| http.credentials.allow.insecure | false | (OkHttp only) If true, send credentials (basic auth, credential headers, cookies) even on connections which do not authenticate the server: cleartext http:// urls and, when `http.trust.everything` is true or `http.verify.hostnames` is false, https:// urls as well. | http.credentials.headers | authorization, proxy-authorization, cookie, x-api-key | (OkHttp only) Header names (case-insensitive) which carry credentials and are therefore withheld from requests to servers which were not authenticated. Applies to the headers configured with `http.custom.headers` and to headers set by request. Names given here are added to the four defaults, which are always treated as credentials. | topology.message.timeout.secs | -1 | OKHTTP message timeout. |=== diff --git a/docs/src/main/asciidoc/extending.adoc b/docs/src/main/asciidoc/extending.adoc index 8b1e1d6fd..25f65ab7e 100644 --- a/docs/src/main/asciidoc/extending.adoc +++ b/docs/src/main/asciidoc/extending.adoc @@ -392,7 +392,7 @@ Keep the pool size (idle + active) under 1000. For large crawls, increase `proto ==== SSL/TLS -By default, StormCrawler's OkHttp protocol validates the certificate chains of HTTPS servers and checks that the certificate matches the host name contacted (`http.trust.everything: false`, `http.verify.hostnames: true`). For crawls of hosts whose certificates cannot be validated, e.g. self-signed ones, set `http.trust.everything` to `true`: the servers are then not authenticated, so credentials (basic auth, credential headers in `http.custom.headers`, cookies) are withheld unless `http.credentials.allow.insecure` is set to `true`. Credentials are also withheld from cleartext http:// requests, which never authenticate the server. Which header names count as credentials is configured with `http.credentials.headers`. Hostname checking is a separate decision and can be disabled with `http.verify.hostnames: false`. +By default, StormCrawler's OkHttp protocol validates the certificate chains of HTTPS servers and checks that the certificate matches the host name contacted (`http.trust.everything: false`, `http.verify.hostnames: true`). For crawls of hosts whose certificates cannot be validated, e.g. self-signed ones, set `http.trust.everything` to `true`: the servers are then not authenticated, so credentials (basic auth, credential headers in `http.custom.headers`, cookies) are withheld unless `http.credentials.allow.insecure` is set to `true`. Credentials are also withheld from cleartext http:// requests, which never authenticate the server. Which header names count as credentials is configured with `http.credentials.headers`. Hostname checking is a separate decision and can be disabled with `http.verify.hostnames: false`; the servers are then not authenticated either, and credentials are withheld in the same way. ==== Authentication @@ -404,7 +404,7 @@ http.basicauth.user: "username" http.basicauth.password: "password" ---- -These credentials are sent as an `Authorization` header with requests to servers authenticated by their TLS certificate; they are withheld from cleartext http:// requests and from https:// requests when `http.trust.everything` is enabled, unless `http.credentials.allow.insecure` is set to `true`. For per-site authentication, use metadata-driven headers instead (see <> in the Internals section). +These credentials are sent as an `Authorization` header with requests to servers authenticated by their TLS certificate; they are withheld from cleartext http:// requests and from https:// requests when `http.trust.everything` is enabled or `http.verify.hostnames` is disabled, unless `http.credentials.allow.insecure` is set to `true`. For per-site authentication, use metadata-driven headers instead (see <> in the Internals section). ==== Proxy Authentication From 6f1c2365f720fe6ce2c88e93e4e77ac24968fd2e Mon Sep 17 00:00:00 2001 From: Richard Zowalla Date: Thu, 10 Sep 2026 13:04:01 +0200 Subject: [PATCH 7/7] Apply code formatting to okhttp HttpProtocol Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014U6Zqh1d6Z9WF6TAUgk3AE --- .../org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java index 5597d9f87..7717be59c 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java @@ -346,7 +346,8 @@ public void configure(Config conf) { hop.tag(ProxyAuthenticated.class) != null && !hop.url().isHttps(); Request.Builder stripped = hop.newBuilder(); for (String name : new HashSet<>(hop.headers().names())) { - if (forProxy && HttpHeaders.PROXY_AUTHORIZATION.equalsIgnoreCase(name)) { + if (forProxy + && HttpHeaders.PROXY_AUTHORIZATION.equalsIgnoreCase(name)) { continue; } if (isCredentialHeader(name)) {